def __init_serv__(ssl, address, port, crt, key, config):
""" This func inits everything. Defines the queue, starts the pool of http workers,
starts the logger thread, bind our script to 0.0.0.0:8080, and finally starts new
thread for each client that connects
"""
# Defines a FIFO queue for requests process
if not config and not address:
exit('Please at least use --address/-i or --config/-C')
elif config:
parsed_config_file = readconfs(config)
address = parsed_config_file['basic']['address']
path = parsed_config_file['basic']['basic_path']
port = int(parsed_config_file['basic']['port'])
ssl = False if parsed_config_file['basic']['ssl'] == 'False' else True
if ssl:
crt = path + 'config/' + parsed_config_file['ssl']['crt']
key = path + 'config/' + parsed_config_file['ssl']['key']
ownqueue = Queue()
logger = init_log()
if ssl:
sock, context = start_ssl_socket(crt, key, server_side=True)
else:
sock, context = start_standard_socket()
# We now don't need workers anymore because we had to adjust for ssl
# We start a pool of N workers
# They are used to serve each requests independently from the source client
# We start the thread that will log every thing into http.log
try:
sock.bind((address, port))
except sock_err as e:
print(
'%s - [INFO] Exception SOCKET_ERROR: Can\'t bind.Please try again later.' %
(time()))
else:
print('%s - [INFO] Server listening on %s:%s' % (time(), address, port))
try:
# Here we just wait until we received a new connection (from a client)
# We then start a thread to handle this specific client / request
sock.listen(0)
for num in range(15):
thread = Child(target=cltthread, args=(logger, sock, context, ssl))
# thread = Process(target=cltthread, args=(queue, logqueue, ownqueue))
# We set each worker to Daemon
# This is important because we can safely ^C now
thread.daemon = True
thread.start()
while True:
# max connection is set to 0, but i guess we could set it to the number of
# started workers. Possibly using threading.activeCount()
sleep(1)
except KeyboardInterrupt:
sock.close()
print('Bye')
exit(0)
import re
import urlparse
import threading
import ConfigParser
import argparse
import time
import logging
import sys
import logger
import config_load
import seedfile_load
import crawl_thread
if __name__ == "__main__":
logger.init_log("./log/spider", level=logging.DEBUG)
# Arguments parser
parser = argparse.ArgumentParser()
parser.add_argument("-v", "--version", help="show current script version", \
action="version", version="%(prog)s 1.0")
parser.add_argument("-c", "--conf", help="set config file", required="true")
args = parser.parse_args()
if args.conf:
conf = args.conf
logging.info("Success to read conf args : %s", conf)
else:
logging.error("Fail to read conf args")
sys.exit(1)
# Config parser
spider_conf = config_load.SpiderConf()
# -*- coding: utf-8 -*-
#
# This file load settings and can modify them
# Settings are store in a JSON file
#
import sys
import os
import json
# Import log #
# sys.path.insert(0, os.path.join(os.path.dirname(__file__), os.pardir, os.pardir))
import logger
#
log = logger.init_log("setting", settings={"log": dict()}, log_type="Console")
VERSION = 1.0
_DEFAULT_SETTING = dict()
_DEFAULT_SETTING["log"] = dict()
_DEFAULT_SETTING["log"]["level"] = "info"
_DEFAULT_SETTING["log"]["output"] = "Console"
_DEFAULT_SETTING["xbee"] = dict()
_DEFAULT_SETTING["xbee"]["baudrates"] = (
0,
2400,
4800,
9600,
19200,
38400,
# I know it's a bit crazy like this but I couldn't come up with a smarter way
# the problem that this has no order
#todo think of a structure where we can specify exactly the cells/cols/rows that could be modified
operations_count = {
'del_row': 0,
'del_col': 0,
'add_row': 0,
'add_col': 0,
'ch_cell': 100, #todo changing in a new row is not considered
# 'me_col': [[0,1], [3,5]], #important to have the array in order ! so not [5,3]!
'me_col': [],
'me_row': [],
# 'me_row': [[0,2,3]],
}
log.init_log(log_file)
result = gen.create_table(rows, cols, min_data, max_data, data_type=int)
#save the input file
gen.save_table(result['table'], result['row_ids'], result['col_ids'], data_directory + in_file_name)
# change
ch_count = 15
for i in range(ch_count):
result = change_table(result, min_data, max_data, operations_count)
#save the output file
gen.save_table(result['table'], result['row_ids'], result['col_ids'], data_directory + file_name + str(i+1) + '_out.csv')
#just print the size to add it manually to index.json
print (result['table'].shape[0],result['table'].shape[1],i)
#update the ... for next round
def main(load, n_tor, n_switches, n_xpand, n_cache, bandwidth, arrive_at_start,
latency, time_limit, workload, slice_duration, reconfiguration_time,
jitter, uuid, log, verbose, no_log, no_pause, skewed, cache_policy,
is_ml, valiant):
# Set parameters
# (Mb/s)*us/8 works out to (B/s)*s
packets_per_slot = int(bandwidth * slice_duration / (BYTES_PER_PACKET * 8))
slice_duration /= 1000 #divide to be in ms
reconfiguration_time /= 1000 #divide to be in ms
bandwidth_Bms = bandwidth * 1e6 / 1e3 / 8
random.seed(40) # TODO Just to make things reproducible
# Compute switch counts
if n_xpand is not None:
assert n_xpand <= n_switches
n_xpand = n_xpand
else:
n_xpand = 0 #round(min(5, n_switches/3))
if n_cache is not None:
assert n_cache + n_xpand <= n_switches
assert n_cache < n_switches
n_cache = n_cache
else:
n_cache = floor((n_switches - n_xpand) / 2)
n_rotor = n_switches - n_xpand - n_cache
print("%d xpander, %d rotor, %d cache. %d total" %
(n_xpand, n_rotor, n_cache, n_switches))
if uuid is None:
uuid = _uuid.uuid4()
slot_duration = slice_duration #*n_rotor
cycle_duration = slice_duration * n_rotor
del slice_duration
PARAMS.set_many(locals())
PARAMS.flow_print = -1
print(PARAMS)
gen_ports()
print("Setting up network...")
# Uses global params object
net = RotorNet()
if n_rotor > 0:
n_slots = math.ceil(time_limit / slot_duration)
n_cycles = math.ceil(time_limit /
(n_rotor * n_slots * PARAMS.slot_duration))
else:
n_cycles = 1
n_slots = 1
max_slots = n_cycles * n_slots
#cycle_duration = slot_duration*n_slots
slice_duration = slot_duration
print("Time limit %dms, cycle %.3fms, slot %.3fms, slice %.3fms" %
(PARAMS.time_limit, PARAMS.cycle_duration, PARAMS.slot_duration,
slice_duration))
print("#tor: %d, #rotor: %d, #links: %d, bw: %dGb/s, capacity: %.3fGb/s" %
(PARAMS.n_tor, PARAMS.n_rotor, PARAMS.n_tor * PARAMS.n_rotor,
PARAMS.bandwidth / 1e3,
PARAMS.n_tor * PARAMS.n_switches * PARAMS.bandwidth / 1e3))
print("Setting up flows, load %d%%..." % (100 * load))
# generate flows
flow_gen = generate_flows(
load=load,
n_tor=n_tor,
bandwidth=bandwidth,
time_limit=time_limit,
n_switches=n_switches,
workload_name=workload,
arrive_at_start=arrive_at_start,
skewed=skewed,
)
# Start the log
if not no_log:
init_log(fn=None, **locals())
# set up printing
time = 0
while time < time_limit * 10:
time += slice_duration
if verbose and not no_pause:
R.call_in(time, pause, priority=100)
#print time
R.call_in(0, print_time, time_limit)
print("Starting simulator...")
if is_ml:
ml_generator(network=net,
n_jobs=3,
servers_per_ring=4,
model_name="resnet")
ml_generator(network=net,
n_jobs=3,
servers_per_ring=4,
model_name="vgg")
ml_generator(network=net,
n_jobs=3,
servers_per_ring=4,
model_name="gpt2")
# Start the simulator
net.run(flow_gen=flow_gen, time_limit=time_limit)
# Force log the unfinished flows
for f in FLOWS.values():
LOG.log_flow_done(f)
# Create a new log with the
u_fn = "utilization-" + str(LOG.sim_id) + ".csv"
max_packets = (R.time / 1000) * (bandwidth * 1e6) / (BYTES_PER_PACKET * 8)
"""
with open(u_fn, "w") as f:
print("switch,type,port,n_packets,divisor", file = f)
for s in net.switches:
#if s.tag == "cache":
#divisor = R.time
#else:
divisor = max_packets
for port_id, n in enumerate(s.n_packets):
print(",".join(str(x) for x in [s.id, s.tag, port_id, n, divisor]), file = f)#
"""
# Done!
if LOG is not None:
LOG.close()
print("done")
# I know it's a bit crazy like this but I couldn't come up with a smarter way
# the problem that this has no order
# todo think of a structure where we can specify exactly the cells/cols/rows that could be modified
operations_count = {
'del_row': 0,
'del_col': 0,
'add_row': 0,
'add_col': 0,
'ch_cell': 100, # todo changing in a new row is not considered
# 'me_col': [[0,1], [3,5]], #important to have the array in order ! so not [5,3]!
'me_col': [],
'me_row': [],
# 'me_row': [[0,2,3]],
}
log.init_log(log_file)
result = gen.create_table(rows, cols, min_data, max_data, data_type=int)
# save the input file
gen.save_table(result['table'], result['row_ids'], result['col_ids'],
data_directory + in_file_name)
# change
ch_count = 15
for i in range(ch_count):
result = change_table(result, min_data, max_data, operations_count)
# save the output file
gen.save_table(result['table'], result['row_ids'], result['col_ids'],
data_directory + file_name + str(i + 1) + '_out.csv')
# just print the size to add it manually to index.json
#!/usr/bin/env python
#-*-coding:utf-8-*-
from db import Connection
from logger import init_log
log = init_log()
class User(Connection):
def __init__(self, **db_setting):
Connection.__init__(self, **db_setting)
def add(self,nickname,gender,desc,friends_count,fans_count,weibo_count,avatar):
try:
sql = """insert into sina_user(nickname,gender,description,friends_count,fans_count,weibo_count,avatar)
values (%s,%s,%s,%s,%s,%s,%s);
"""
self.execute(sql,nickname,gender,desc,friends_count,fans_count,weibo_count,avatar)
except Exception as e:
print e
def get_user_by_condition(self):
sql = "select nickname,gender,avatar from sina_user where friends_count < 30 and fans_count < 30"
return self.query(sql)
class Comment(Connection):
def __init__(self, **db_setting):
Connection.__init__(self, **db_setting)
import psutil
import socket
import sys
import threading
import time
import urllib2
try:
import psutil
except ImportError as e:
print "You should 'pip install psutil' firstly"
sys.exit()
qos_result = {}
sys_load_result = {}
logger.init_log("route_map")
def get_local_lr_id(lrid_file):
"""
Get local LR id from local file
"""
try:
with open(lrid_file) as fd:
local_lr_id = fd.read()
except IOError:
logger.logger.error("No such file %s" % lrid_file)
local_lr_id = ''
return local_lr_id.strip()
#!/usr/bin/env python
# coding=utf-8
"""
主程序，运行入口。
"""
import argparse
import logging
import sys
from logger import init_log
from config_parse import ConfigSpiderConf
from crawler import MyCrawler
if __name__ == "__main__":
init_log("./log/mini_spider")
logging.info(u"开始解析参数...")
parser = argparse.ArgumentParser(
description="Mini Spider of Multithreading.")
parser.add_argument("-v", action="store_true", help="Show Version info.")
parser.add_argument("-c", action="store", help="Input config file.")
args = parser.parse_args()
if args.v is True:
print "Mini Spider V 1.0."
logging.info(u"解析参数结束.")
sys.exit(0)
if args.c is None:
logging.info(u"解析参数结束.")
sys.exit(1)
logging.info(u"解析参数结束.")
def start(self):
"""
启动爬虫
"""
# # 进程池中的进程的个数最大为配置文件中指定的process_count, 在不超过该数值的前提下，为每一个种子开启一个进程
# process_pool_size = self.conf.process_count if self.conf.process_count < len(self.seeds) else len(self.seeds)
# process_pool = multiprocessing.Pool(processes=process_pool_size)
# # 多进程并行爬取每个种子
# # 每个种子内部使用多线程并发处理抓取和存储等IO密集型任务
# process_pool.map(self.traverse_seed, self.seeds)
# process_pool.terminate()
# process_pool.join()
# logging.info(f'All subprocess of top level seeds complete')
self.traverse_seed(self.seeds)
if __name__ == '__main__':
# 初始化符合百度log规范的logger,将log输出到同级目录log文件夹下的mini_spider.log
logger.init_log("./log/mini_spider")
arger_parser = argparse.ArgumentParser()
arger_parser.add_argument('-c',
help='set spider_test.conf file path',
dest='config_path',
required=True)
arger_parser.add_argument('-v', help='set spider version', dest='version')
args = arger_parser.parse_args()
spider = MiniSpider(args.config_path)
spider.start()
# -*- coding: utf-8 -*-
#
import serial
import threading
import time
import numpy as np
import mtools
from settings import settings
import logger
log = logger.init_log("wsn")
class Xbee(object):
"""
This class represent the Xbee module
"""
def __init__(self, serial_path=None, ATMY=None, ATID=None, ATCH=None, ATBD=None):
"""
This method initiate the Xbee object but not the Xbee
:return:
"""
self._lock_serial = threading.Lock()
if serial_path is None:
serial_path = str(settings.get("xbee", "serial"))
if ATBD is None:
ATBD = str(settings.get("xbee", "ATBD"))
# -*- coding: utf-8 -*-
# This module treat frames to send DMX data
import threading
import numpy as np
import ola.ClientWrapper
import math
import time
from settings import settings
from logger import init_log
log = init_log("dmx")
class DmxManager(object):
"""
"""
def __init__(self, universe=0, dmxoutput_size=255):
"""
:param universe:
:param dmxoutput_size:
:return:
"""
self.universe = universe
self.wrapper = ola.ClientWrapper.ClientWrapper()
self.client = self.wrapper.Client()
self.client.SendDmx(self.universe, np.zeros(255, dtype=np.uint8)) # Clean all the universe
def main():
screen_dims = (240, 160)
pygame.init()
pygame.display.set_caption('GBA Freecell')
screen = pygame.display.set_mode(screen_dims)
controller = Controller() # Input device
held_dpad_direction = None
dpad_repeat_event = pygame.USEREVENT + 0
dpad_repeat_start_delay = 250
dpad_repeat_delay = 50
logger.init_log()
clock = pygame.time.Clock()
fps = 0
deal_event = pygame.USEREVENT + 1
c_transparent = pygame.Color('#ff00ff')
background = pygame.Surface(screen_dims)
board_bmp = pygame.image.load('board.bmp')
background.blit(board_bmp, (0, 0))
global INPUT_ENABLED
global GAME_IN_PROGRESS
global MENU_OPEN
global DEALING
INPUT_ENABLED = False
GAME_IN_PROGRESS = True
DEALING = True
MENU_OPEN = False
# Card dims: 19 x 28
deck_pos = (screen_dims[0] / 2 - 10, screen_dims[1] - 28 - 6)
card_back = pygame.image.load('card_back.bmp')
card_back.set_colorkey(c_transparent)
suits = ('spades', 'clubs', 'diamonds', 'hearts')
cells = [Cell(cell_type='cell', pos=n, col=0) for n in range(4)]
foundations = [
Cell(cell_type='foundation', pos=n, col=9, suit=suits[n])
for n in range(4)
]
bases = [Cell(cell_type='base', pos=n, col=n + 1) for n in range(8)]
cards = []
for val in range(1, 14):
for suit in suits:
cards.append(
Card(pos=deck_pos,
transparent=c_transparent,
value=val,
suit=suit,
suits=suits))
board = Board(cards, foundations, cells, bases, c_transparent)
board.shuffle()
board.initialize_card_cols()
board.initialize_card_target_positions()
board.deal(deal_event)
board.reset_tableaux()
is_running = True
while is_running:
clock.tick(60)
fps = clock.get_fps()
for event in pygame.event.get():
if event.type == pygame.QUIT:
is_running = False
elif event.type == deal_event:
board.deal(deal_event)
elif event.type == dpad_repeat_event:
try:
held_dpad_button = [
b for b in controller.dpad if b['pressed']
][0]
direction = held_dpad_button['name'].split(' ')[-1].lower()
board.handle_move_hover(direction=direction)
pygame.time.set_timer(dpad_repeat_event, dpad_repeat_delay,
True)
except IndexError:
held_dpad_direction = None
elif event.type in (pygame.KEYDOWN, pygame.KEYUP):
input_event = None
if INPUT_ENABLED:
input_event = controller.get_action_button(event)
if input_event == 'A press':
if board.selected_card:
# Should always be over a valid move
board.place_selected_card()
board.set_cards_z_index()
else:
# Should always be over a valid card to move
board.select_hovered()
elif input_event == 'B press':
if board.selected_card:
board.deselect()
elif input_event == 'D-PAD UP press':
board.handle_move_hover(direction='up')
pygame.time.set_timer(dpad_repeat_event,
dpad_repeat_start_delay, True)
elif input_event == 'D-PAD RIGHT press':
board.handle_move_hover(direction='right')
pygame.time.set_timer(dpad_repeat_event,
dpad_repeat_start_delay, True)
elif input_event == 'D-PAD DOWN press':
board.handle_move_hover(direction='down')
pygame.time.set_timer(dpad_repeat_event,
dpad_repeat_start_delay, True)
elif input_event == 'D-PAD LEFT press':
board.handle_move_hover(direction='left')
pygame.time.set_timer(dpad_repeat_event,
dpad_repeat_start_delay, True)
elif input_event in ('D-PAD UP release', 'D-PAD RIGHT release',
'D-PAD DOWN release',
'D-PAD LEFT release'):
held_dpad_direction = None
elif input_event == 'SPACE press':
toggle_menu()
# Draw background (board)
screen.blit(background, (0, 0))
# Draw cards
for card in board.cards:
if not DEALING:
board.update_highlights()
card.update()
screen.blit(card.surf, card.pos)
# Draw hover markers, if applicable
if board.selected_card:
for n in range(len(board.hover_markers)):
screen.blit(board.hover_markers[n],
(board.hover_marker_positions[n]))
# Draw card back 'deck' while dealing
if DEALING:
if [
c for c in board.cards
if c.pos != c.target_pos and not c.animating
]:
screen.blit(card_back, deck_pos)
# Unset DEALING flag when cards are done animating
cards_animating = [c for c in board.cards if c.animating]
if not cards_animating:
DEALING = False
INPUT_ENABLED = True
# Select bottom card in first cascade
board.hover_last_card_in_cascade(1)
pygame.display.update()