class Util(object):
'''
工具类，这么个破东西，整个还像个样子，doubi
'''
config = Configure()
_m_days_o = [31, 28, 31, 30, 31, 30 ,31, 31, 30, 31, 30, 31]
_m_days_n = [31, 29, 31, 30, 31, 30 ,31, 31, 30, 31, 30, 31]
@staticmethod
def get_time():
if Util.config.get_time_type() == 0:
return Util._get_system_time()
else:
return Util._get_network_time()
@staticmethod
def get_day_before(cur_day):
if len(cur_day) != 3:
return cur_day
day = int(cur_day[2])
mon = int(cur_day[1])
year = int(cur_day[0])
new_day = cur_day[:]
if day != 1:
new_day[2] = str(day - 1)
return new_day
days = Util._get_days(cur_day)
new_day = cur_day[:]
if mon-1 != 0:
new_day[1] = str(mon-1)
new_day[2] = str(days[mon-1])
else:
new_day[0] = str(year-1)
new_day[1] = '12'
new_day[2] = '31'
return new_day
@staticmethod
def _get_days(cur_day):
if len(cur_day) != 3:
return Util._m_days_o
year = int(cur_day[0])
if (year%4 == 0 and year%100 != 0 ) or year%400 == 0:
return Util._m_days_n
return Util._m_days_o
@staticmethod
def _get_system_time():
ltime = time.localtime()
if ltime != None:
return [str(ltime.tm_year),"%02d" % ltime.tm_mon, "%02d" % ltime.tm_mday]
else:
return None
@staticmethod
def _get_network_time():
'''
Currently we just use local time
Otherwise we should specify an timezone
'''
return Util._get_system_time()
def wrapper(*args, **kwargs):
cfg = Configure('./config/config.cfg')
mongo_url = cfg.get_config('mongodb', 'url')
mongo_port = int(cfg.get_config('mongodb', 'port'))
dbname = cfg.get_config('mongodb', 'dbname')
collection_name = cfg.get_config('mongodb', tablename)
client = MongoClient(mongo_url, mongo_port)
db=client[dbname]
collection = db[collection_name]
res = func(*args, c = collection, **kwargs)
return res
def __init__(self):
self.config = Configure()
self.start_server, self.client_ip, self.server_name, self.server_pass = self.config.read_server(
)
if self.start_server == False:
sys.exit()
say('Starting Server')
self.server = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.host = ''
self.port = 5150
self.server.bind((self.host, self.port))
self.server.listen(5)
print('Listing for a client...')
self.client, self.addr = self.server.accept()
print('Accepted connection from ', self.addr)
self.client.send(str.encode('Connection made with Andy'))
def calculate_index():
print '%s\tstart update index' % str(datetime.now())
cfg = Configure('./config/config.cfg')
article_repo = cfg.get_config('running', 'article_repo')
file_names = os.listdir(
os.path.join(article_repo,
datetime.today().strftime('%m%d%Y')))
doc_count = 0
for filename in file_names:
if not filename.endswith('_DONE'):
process(os.path.join(article_repo, filename), filename)
os.rename(os.path.join(article_repo, filename),
os.path.join(article_repo, filename + '_DONE'))
doc_count += 1
if doc_count:
update_tfidf(doc_count)
print '%s\tend update index' % str(datetime.now())
def __init__(self, path, default, quiet):
'''Constructs the bot object. Takes path, default, and quiet arguments from the command
line input and sets the bot accordingly. Initializes logging, creates instances of
necessary classes. Loads plugins, begins the connection.'''
self._config_path = path
self._default = default
self._quiet = quiet
self.logger = logging.getLogger("GorillaBot")
self._configuration = Configure(self._config_path, self._default,
self._quiet)
settings = self._configuration.get_configuration()
self.GorillaConnection = Connection(
self, settings["host"], settings["port"], settings["nick"],
settings["ident"], settings["realname"], settings["chans"],
settings["botop"], settings["fullop"])
self.GorillaCommander = CommandManager(self, self.GorillaConnection)
self.GorillaConnection._connect()
def freezeService(code):
c = Configure()
c.setStatus({
'message': '发生异常，系统自动冻结服务。',
'title': '服务已冻结',
'me': False,
'tomorrow': True,
'today': True,
'mon': False,
'detect': False,
'isRed': True
})
data = {
'service_name': {
'value': '全服务'
},
'reason': {
'value': '网络异常：code:' + code
},
'mess': {
'value': '服务已自动冻结'
}
}
pushRunningMessage('warn',data)
from feature_selection import FeatureSelection
from config import Configure
import pandas as pd
print('\n Aplicando algoritmo de seleção de parâmetros')
settings = Configure()
settings.set_fs_params()
settings.set_pre_processing_params()
pp_params = settings.pre_processing_params
fs_params = settings.feature_selection_params
df1 = pd.read_csv(settings.pf1_folder)
df2 = pd.read_csv(settings.pf2_folder)
df3 = pd.read_csv(settings.pf3_folder)
mkt = pd.read_csv(settings.mkt_folder)
fs = FeatureSelection(mkt, df1, df2, df3, pp_params, fs_params)
values, features = fs.feature_selection_algorithm(m='RFECV')
columns = features.values[features.values != 'Unnamed: 0']
mkt = mkt[columns]
def __init__(self):
self.config = Configure()
self.errors = []
self.meta = {'info':'', 'real_url':'', 'code':0}
sys.setdefaultencoding('utf-8')
import os
os.chdir(os.path.dirname(os.path.realpath(__file__)))
import hashlib
import time
import newspaper
import feedparser
from datetime import datetime
from random import shuffle
from config import Configure
from util import Util
CFG = Configure('./config/config.cfg')
ARTICLEREPO = os.path.join(CFG.get_config('running', 'article_repo'),
datetime.today().strftime('%m%d%Y'))
if not os.path.isdir(ARTICLEREPO):
os.makedirs(ARTICLEREPO)
def crawl_web_categorypage(url):
paper = newspaper.build(url, memoize_articles=False, language='en')
for category in paper.category_urls():
crawl_web(category)
def crawl_web(url):
paper = newspaper.build(url, memoize_articles=False, language='en')
"""
Disable all elements while operations working.
"""
self.btn_fill.setDisabled(True)
self.btn_exp.setDisabled(True)
self.btn_scrapp.setDisabled(True)
self.btn_save.setDisabled(True)
self.cmb_ds.setDisabled(True)
self.cmb_exp.setDisabled(True)
self.cmb_save.setDisabled(True)
self.page_count_box.setDisabled(True)
if __name__ == '__main__':
logging.basicConfig(level=logging.DEBUG)
logging.debug("[+]App started...")
# Configure main program params...
cfg = Configure()
app = QtWidgets.QApplication(sys.argv)
form = MainForm(config=cfg)
form.show()
logging.debug("[+]App set_enable...")
sys.exit(app.exec_())
from config import Configure
__author__ = "Andrew Gafiychuk"
if __name__ == '__main__':
# Test Singleton also
cfg2 = Configure(file="../config.cfg")
print(cfg2._singleton_mark)
cfg = Configure(file="../config.cfg")
print(cfg._singleton_mark)
print(cfg2 is cfg) # Must be a True
cfg.show_config()
config = cfg.getConfiguration()
print("=" * 100)
print("User: {0}".format(config["user"]))
print("Passowrd: {0}".format(config["password"]))
print("=" * 100)
print("Host: {0}".format(cfg.get_host()))
print("Port: {0}".format(cfg.get_port()))
def __init__(self):
self.serverName, self.serverPort, self.clientPort, self.downloadPath = Configure(
).client_config()
from config import Configure
from model import Net
from data_process import data_loader
import torch
import torch.nn as nn
config = Configure()
args = config.get_args()
train_set, val_set, num_total_words = data_loader(args)
device = torch.device("cuda:2" if torch.cuda.is_available() else "cpu")
net = Net(num_total_words*args.max_features, args.embedding_size,
args.num_hiddens, args.num_layers, num_classes=len(args.catogories))
net.to(device)
criterion = nn.CrossEntropyLoss()
optimizer = torch.optim.SGD(net.parameters(), lr=args.lr)
for epoch in range(args.num_epochs):
train_set, val_set, num_total_words = data_loader(args)
train_loss, val_loss = 0., 0.
train_acc, val_acc = 0., 0.
num_correct = 0
for step, (feature, label) in enumerate(train_set):
feature = torch.from_numpy(feature).to(device)
label = torch.from_numpy(label).to(device)
net.zero_grad()
scores = net(feature)
def main(DEBUG=False):
#Start LEDs
write('load')
subprocess.Popen(['python', 'LED.py'])
#Read from configuration files and apply settings.
config = Configure()
start_andy_on_boot, social_mode, rebel_mode, wander_mode = config.read()
if start_andy_on_boot == False:
sys.exit()
#Say start up slogan and check for command.
say('Hello. My name is Andy. Please wait while my system starts up.')
getit = Get_cmd()
#Start server process
subprocess.Popen(['python', 'server.py'])
write(None)
#Enter main loop
while True:
print()
#Get the command and convert it to a list
cmd = getit.get().split()
write('load')
if DEBUG: print('COMMAND:', cmd)
#Remove unused words from command
num = 0
for word in cmd:
cmd[num] = word.lower()
num += 1
if len(cmd) == 0:
cmd.append('')
#Determin master command
if cmd[0] == 'what':
cmds.What(cmd, DEBUG)
elif cmd[0] == 'walk' or cmd[0] == 'turn':
cmds.Walk(cmd, DEBUG)
elif cmd[0] == 'stop':
cmds.Walk(cmd, DEBUG).stop()
elif cmd[0] == 'pickup' or cmd[0] == 'pick' and cmd[1] == 'up':
cmds.Arm(cmd, DEBUG).pickup()
elif cmd[0] == 'set' and cmd[1] == 'down':
cmds.Arm(cmd, DEBUG).setdown()
elif cmd[0] == 'where':
cmds.Where(cmd, DEBUG)
elif cmd[0] == 'take':
cmds.Take(cmd, DEBUG)
elif cmd[0] == 'set':
pass #TODO
elif cmd[0] == 'tell':
cmds.Tell(cmd, DEBUG)
elif cmd[0] == 'who':
cmds.Who(cmd, DEBUG)
elif cmd[0] == 'shutdown':
cmds.shutdown()
elif cmd[0] == 'sleep':
cmds.sleep()
elif cmd[0] == 'meet':
Meet(cmd, DEBUG)
else:
say('Not valid command')
write(None)
time.sleep(1)
def set_manual_transformation(self):
settings = Configure()
settings.set_pre_processing_params()
rules = settings.pre_processing_params['manual_encoding']
return rules
elif adhoc_interface == "ath1" and monitor_interface == "ath0":
working_iface_monitor = re.compile('[1]').sub(
'3', adhoc_interface)
backup_iface_monitor = re.compile('[0]').sub(
'2', monitor_interface)
else:
print "Please specify [ath0, ath1] for each interface. "
sys.exit(0)
print "Current Interfaces: [wa: %s] [wm: %s] [ba: %s] [bm: %s]" % (
working_iface_adhoc, working_iface_monitor, backup_iface_adhoc,
backup_iface_monitor)
print "Current Interface: [wa: ath%s] [ba: ath%s]" % (working,
backup)
cf = Configure(working_iface_adhoc, backup_iface_adhoc)
ff = FrameFilter(cf, snr_threshold, FILTER)
p = pcap.pcapObject()
p.open_live(backup_iface_monitor, 96, 0, 100)
while 1:
while ff.rx_frame < 101: # Approx. 100ms * 100 = 10s ; Only beacon frames counted
apply(ff.filter, p.next())
#ff.print_tx_filter(working_iface_adhoc) # maybe 1s
stime = time.time()
#print "loop starts %f" % stime
try:
for daddr in ['00:80:92:3e:18:11', '00:80:92:3e:18:18']:
print "XXXXXXXXXXXXXXXX"
def initConf(file):
try:
conf = Configure(file)
return conf
except Exception as ex:
raise ex
from data.dataloader import generate_loader
from config import Configure
from networks.RCNN import BiGruCNN
from framework import MyFrame
from utils import initLogging
name = 'HAN_rcnn_drop0.2'
mylog = 'logs/' + name + '.log'
path = 'weights/' + name + '.pkl'
initLogging(mylog)
device = 2
total_epochs = 30
valid_best_score = 0.
train_loader, valid_loader, testa_loader = generate_loader(train_bs=32)
opt = Configure()
net = BiGruCNN
loss_func = CrossEntropyLoss(size_average=True)
solver = MyFrame(net=net, loss=loss_func, opt=opt, lr=1e-3, device=device)
solver.load(path)
# solver.net.embedding.weight.requires_grad = True
no_optim_round = 0
for epoch in range(total_epochs):
# train
solver.train_mode()
train_loss = 0.
for X, y in tqdm(train_loader):
solver.set_input(X, y)
step_loss = solver.optimize()
train_loss += step_loss
