def gameExists(self, uniqueurl):
for i in range(0, len(self.sessions)):
if self.sessions[i][0] == int(uniqueurl):
Logger.log('Game {0} exists'.format(uniqueurl))
return i
Logger.warn('Game {0} doesn\'t exist'.format(uniqueurl))
return None
def __main__():
logger = Logger()
logger.set_debug_level('info')
config= Config()
u_base_path = config.base_path
hddb = HoerdatDB(config.hoerdat_db_path, {'debug': 0})
dupmerge = DupMerge(
config.md5_db_path, u_base_path,
{'logger': logger},
)
hd = Hoerdat(hddb)
for file, md5 in dupmerge.all_files_md5():
if not hd.set_filename(file, md5):
continue
if DONT_RESCAN_PRESENT and hd.data['hoerdat'].get_data('hoerdat_id'):
continue
print file, hd.data['hoerdat'].get_data('hoerdat_id')
fetch_result = hd.fetch_data()
best_match = hd.get_best_match(fetch_result)
if best_match:
hd.merge_data(best_match[0], best_match[1])
hd.update()
print "Best rank:", best_match[1]
print "\t", '"' + hd.data['hoerdat'].get_data('title') + '"'
def _configure(self): """ Set members based using inventory. """ Logger._configure(self) self.includeDealloc = self.inventory.includeDealloc return
def _add_player(self):
"""
Dodaje połączenie gracza do serwera.
Wchodzi w to sprawdzanie, czy może się połączyć (serwer nie jest pełen),
oraz odpowiednie zapisanie i rejestrację danych po stronie serwera.
"""
client_sock, _ = self._sock.accept()
if len(self._client_sockets) == 2:
client_sock.close()
return
client_sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
self._register_in_poll(client_sock)
pl = 0
for key in self._player_bindings:
if self._player_bindings[key] == 0:
pl = 1
break
self._player_bindings[client_sock.fileno()] = pl
self._proto.push_player(client_sock.fileno(), pl+1)
if len(self._client_sockets) == 2:
Logger.log('Mamy dwóch graczy, startujemy grę!')
self._gamestate.start()
self._proto.push_start()
class BtObjectPersister:
def __init__(self, persister, torrentIdent):
self.persister = persister
self.torrentIdent = torrentIdent
self.log = Logger('BtPersister', '%-6s - ', torrentIdent)
def load(self, key):
return self.persister.load(self.torrentIdent+'-'+key)
def get(self, key, default=None):
return self.persister.get(self.torrentIdent+'-'+key, default)
def store(self, key, obj, encoding='base64', sync=False):
self.persister.store(self.torrentIdent+'-'+key, obj, encoding, sync)
def remove(self, key, strict=False, sync=False):
self.persister.remove(self.torrentIdent+'-'+key, strict, sync)
def removeAll(self):
btKeys = self.persister.keys('^'+self.torrentIdent+'-')
for key in btKeys:
self.log.debug('Removing key "%s" from persister', key)
self.persister.remove(key)
class WikiCrawler(object):
TAG = "WikiCrawler"
def __init__(self):
self.queue = QueueProxy()
self.entryMgr = EntryManager()
self.catagoryMgr = CatagoryManager()
self.logger = Logger()
def run(self):
while True:
try:
sleep(1)
url = self.queue.pop()
if self.entryMgr.contains(url):
self.entryMgr.update(url)
else:
html = HtmlReader(url).read()
entry = HtmlParser(html, url).parse()
self.entryMgr.add(entry)
self.catagoryMgr.add(entry)
for child in entry.children:
self.queue.push(child)
except Exception, e:
self.logger.e(self.TAG, str(e))
def work(self, imageProcessor, filters):
if not imageProcessor:
raise Exception('No image processor for work')
imageNumbersForProcess = self._getImageNumbersForProcess()
Logger.write('end explode video. images count=%d' % len(imageNumbersForProcess))
imageProcessor.process(imageNumbersForProcess, filters)
class PointLabel:
def __init__(self, df: pd.DataFrame, ax, fig, x_axis: str, y_axis: str,
labels: list, file_=None):
self.df = df.copy()
self.axis = ax
self.fig = fig
self.x_axis = x_axis
self.y_axis = y_axis
self.labels = labels
if 'score' in self.labels:
self.labels.remove('score')
self.has_written_title = False
if file_ is not None:
self.file_handler = file_
else:
self.file_handler = Logger('point_label.log')
def onpick(self, event):
"""
inspired by http://matplotlib.org/examples/event_handling/pick_event_demo.html
"""
ind = event.ind[0]
row = self.df.iloc[ind]
if not self.has_written_title:
self.file_handler.log('picker %s %s %s description' %
(self.x_axis, self.y_axis, '\t'.join(label for label in self.labels if label != 'description')), skip_stamp=True)
self.has_written_title = True
self.file_handler.log('picker %.2f\t%.2f\t%s %s' %
(row[self.x_axis], row[self.y_axis],
'\t'.join("%.2f" % row[label] for label in self.labels if label != 'description'),
row['description']), skip_stamp=True)
def __init__(self):
QtGui.QDialog.__init__(self)
self.setupUi(self)
self.progressBar.setFormat('%p%')
self.progressBar.setValue(0)
# ic=()
#datafromsamson={'FAM': u'surname', 'IM': u'firstname', 'OT': u'parentname'}
#todbf={}
ImportConfig.parse()
# samsmgr=SamsonManager()
# samsmgr.ManageRawKmivcData(KmivcManager.Getrawkmivcdata())
#for n in range(0,50,1):
# itme=QtGui.QListWidgetItem('test'+str(n))
# self.listWidget.addItem(itme)
self.samsonadr.setText(ConnectionsManager.samsonconn['ip'])
self.samsonbd.setText(ConnectionsManager.samsonconn['db'])
self.samsonlogin.setText(ConnectionsManager.samsonconn['login'])
self.samsonpassword.setText(ConnectionsManager.samsonconn['password'])
#connect(lineEdit,SIGNAL(textChanged(QString)),this,SLOT(textChangedSlot(QString)));
QtCore.QObject.connect(self.samsonadr, QtCore.SIGNAL("textChanged(QString)"), self.on_samsonadr_changed)
QtCore.QObject.connect(self.samsonbd, QtCore.SIGNAL("textChanged(QString)"), self.on_samsonbd_changed)
QtCore.QObject.connect(self.samsonlogin, QtCore.SIGNAL("textChanged(QString)"), self.on_samsonlogin_changed)
QtCore.QObject.connect(self.samsonpassword, QtCore.SIGNAL("textChanged(QString)"), self.on_samsonpassword_changed)
Logger.log(u'Программа запущена')
def step_impl(context, element_name):
element = world.find_element(element_name)
if element.wait_for_attribute('selected', False):
return True
else:
log.failed("Verify element '%s' unchecked?" % element.name, "checked", "unchecked")
class RequestBodyReciever(Protocol):
"""needed to actually send the response from a server (because of the way the response object works).
Passes any data it recieves to the Peer writer. This is an unfortunate side effect of the twisted
architecture. A response object cannot pass it's body onwards without the use of this mitigating class"""
def __init__(self,pClient,range):
self.pClient = pClient #reference to persistent client class that holds an
#open TCP connection with the peer
self.recvd = 0
self.start = range[0] #placeholder for a deferred callback (incase one is eventually needed)
self.size = range[1] - range[0]
self.log = Logger()
def repeatCallback(self):
try:
range = self.pClient.father.getNextChunk(self.pClient.id)
if range != None:
self.pClient.getChunk(range)
except:
self.log.warning('error in repeat callback on dlp')
def dataReceived(self,bytes):
self.recvd += len(bytes)
self.pClient.father.appendData(self.pClient,self.start,bytes)
def connectionLost(self,reason):
if self.recvd < self.size:
#server sent back a splash page or something other then the desired content
self.pClient.father.endSession("Mismatched response length from server")
self.log.info("Server ended transmission successfully with local pClient")
self.repeatCallback()
def push_all(self, packet):
"""
Wrzuca na wszystkie kolejki pakietów odpowiedni pakiet do wysłania.
"""
Logger.log('Puszczamy pakiet WSZYSTKIM...')
for client in self._clients:
self._queues[client].append(packet)
def join(uniqueurl):
Logger.printCurrentState(
self.RequestHandler.SessionStorage.sessions)
Logger.log('Received request on /joingame')
res = self.RequestHandler.dispatchRequest(
Request(requestKind=RequestKind.JoinGameRequest, uniqueurl=uniqueurl))
return str(res)
def __init__(self, x_init, y_init, mass, radius, borders):
Logger.debug("KINEMATICS: PhysicsObject init(x_init=%s, y_init=%s, mass=%s, radius=%s, borders=%s)", str(x_init), str(y_init), str(mass), str(radius), str(borders))
self._pos = Vector(x_init, y_init)
self._mass = mass
self._radius = radius
self._vel = Vector(0, 0)
self._borders = borders
def step_impl(context):
alert = world.get_alert_when_exist()
if alert is not None:
alert.dismiss()
else:
log.failed("Fail to cancel popup alert")
def step_impl(context):
alert = world.get_alert_when_exist()
if alert is not None:
alert.accept()
else:
log.failed("Fail to accept popup alert")
def step_impl(context, element_name):
element = world.find_element(element_name)
if element.value == "":
return True
else:
log.failed("Verify value is empty?", element.value, "<blank>")
def step_impl(context, element_name):
element = world.find_element(element_name)
if element.value == context.text:
return True
else:
log.failed("Verify value is? (multi-line)", element.value, context.text)
def step_impl(context, element_name):
element = world.find_element(element_name)
if context.text in element.value:
return True
else:
log.failed("Verify value contains? (multi-line)", element.value, value)
def step_impl(context, element_name):
element = world.find_element(element_name)
if element.wait_for_not_exist():
return True
else:
log.failed("Verify element '%s' exists?" % element.name, "exists", "not exists")
def step_impl(context, element_name):
element = world.find_element(element_name)
if element.wait_for_attribute('visible', True):
return True
else:
log.failed("Verify element '%s' visible?" % element.name, "not visible", "visible")
def step_impl(context, element_name):
element = world.find_element(element_name)
if element.wait_for_attribute('enabled', False):
return True
else:
log.failed("Verify element '%s' disabled?" % element.name, "enabled", "disabled")
def __main__():
config= Config()
u_base_path = config.base_path
logger = Logger(config.ftp_log_path)
logger.set_debug_level('info')
dupmerge = DupMerge(
config.md5_db_path, u_base_path,
{'logger': logger},
)
ftp = FTPFetch(
host = config.ftp_hostname,
port = config.ftp_port,
credentials = config.ftp_credentials,
params = {
'dstdir': config.ftp_base_path,
'db': config.ftp_db_path,
'ignore': config.ftp_ignore_list,
'possible_hidden_dirs': config.ftp_possible_hidden_dirs,
'dupmerge': dupmerge,
'reverse': 0,
'old_cleanup': 1,
'logger': logger,
}
)
ftp.link_all_duplicates()
def step_impl(context, element_name, value):
value = world.replace_variables(value)
element = world.find_element(element_name)
if value in element.value:
return True
else:
log.failed("Verify value contains?", element.value, value)
def step_impl(context, element_name, value):
value = world.replace_variables(value)
element = world.find_element(element_name)
if element.get_attribute('title') == value:
return True
else:
log.failed("Verify tooltip text is?", element.title, value)
def main():
start_time = time.perf_counter()
# Setup logger
log_to_console = '-p' in sys.argv
logger = Logger('log.txt', log_to_console)
# Get urls of the community api files from the api directory
ff_api_urls = get_ff_api_urls(logger)
# Get the urls to the nodelist and nodes_json files
ff_urls = UrlSet()
nodelist_urls = UrlSet()
threads = []
for i in range(0, NUMBER_OF_THREADS):
thread = Thread(target=get_map_urls, args=(ff_api_urls, logger, ff_urls, nodelist_urls))
threads.append(thread)
thread.start()
for thread in threads:
thread.join()
nodes_json_urls = get_nodes_json_urls(ff_urls)
# Get the nodes from the nodes_json_urls
nodes = NodeDict()
threads = []
for i in range(0, NUMBER_OF_THREADS):
thread = Thread(target=get_nodes_from_nodes_json_urls, args=(nodes_json_urls, nodes, logger))
threads.append(thread)
thread.start()
for thread in threads:
thread.join()
# Get the nodes from the nodelist_urls
threads = []
for i in range(0, NUMBER_OF_THREADS):
thread = Thread(target=get_nodes_from_nodelist_urls, args=(nodelist_urls, nodes, logger))
threads.append(thread)
thread.start()
for thread in threads:
thread.join()
# Write found nodes in a json file.
data = dict()
data['timestamp'] = int(time.time())
nodes_out = nodes.getNodes()
logger.log(str(len(nodes_out)) + " nodes found.")
data['nodes'] = nodes_out
write_json_to_file(data)
end_time = time.perf_counter()
duration = end_time - start_time
logger.log('Elapsed time: ' + str(duration))
class Cotorra( object ):
def __init__(self):
self.config = Configurator('/etc/core/etc/cotorra.cfg')
self.logger = Logger(FILE_LOG)
def main(self):
stdin = sys.stdin.readlines()
self.logger.addLine('Veo el correo')
def step_impl(context, element_name, value):
value = world.replace_variables(value)
element = world.find_element(element_name)
classes = element.get_attribute('class')
if value in classes:
return True
else:
log.failed("Verify class contains?", classes, value)
def __init__(self, name="memory_logger"):
"""
Constructor.
"""
Logger.__init__(self, name)
self.megabyte = float(2**20)
self.memory = {}
self.memory['Completion'] = 0
return
def __init__(self, image, imagerect, position):
"""
:param image: shape or image to be drawn
:param position: position on screen (of type Point)
:param imagerect: rectangle of image to be displayed
"""
Logger.debug("DRAWABLE: init(position=(%s)", str(position))
self._image = image
self._pos = position
def __init__(self, datasetPath, testPath, dictionaryPath):
self.datasetPath = datasetPath
self.testPath = testPath
self.dictionary = dictionaryPath
self.logger = Logger(self.logLevel)
def main():
global args, best_prec1, logger
args = parser.parse_args()
os.makedirs(args.prefix)
os.makedirs(os.path.join(args.prefix, 'tensorboard'))
logger = Logger(os.path.join(args.prefix, 'tensorboard'))
# create model
if args.pretrained:
print("=> using pre-trained model '{}'".format(args.arch))
model = models.__dict__[args.arch](pretrained=True)
else:
print("=> creating model '{}'".format(args.arch))
model = models.__dict__[args.arch]()
if torch.cuda.is_available():
model = torch.nn.DataParallel(model).cuda()
# define loss function (criterion) and optimizer
criterion = nn.CrossEntropyLoss()
if torch.cuda.is_available():
criterion = criterion.cuda()
optimizer = torch.optim.SGD(model.parameters(),
args.lr,
momentum=args.momentum,
weight_decay=args.weight_decay)
# optionally resume from a checkpoint
if args.resume:
if os.path.isfile(args.resume):
print("=> loading checkpoint '{}'".format(args.resume))
checkpoint = torch.load(args.resume)
args.start_epoch = checkpoint['epoch']
best_prec1 = checkpoint['best_prec1']
model.load_state_dict(checkpoint['state_dict'])
optimizer.load_state_dict(checkpoint['optimizer'])
print("=> loaded checkpoint '{}' (epoch {})".format(
args.resume, checkpoint['epoch']))
else:
print("=> no checkpoint found at '{}'".format(args.resume))
cudnn.benchmark = True
# Data loading code
traindir = os.path.join(args.data, 'train')
valdir = os.path.join(args.data, 'test')
normalize = transforms.Normalize(mean=[0.4933, 0.5221, 0.5547],
std=[0.2522, 0.2433, 0.2646])
train_dataset = datasets.ImageFolder(
traindir,
transforms.Compose([
transforms.RandomResizedCrop(224),
transforms.RandomHorizontalFlip(),
transforms.ToTensor(),
normalize,
]))
train_loader = torch.utils.data.DataLoader(train_dataset,
batch_size=args.batch_size,
shuffle=True,
num_workers=args.workers,
pin_memory=True)
val_loader = torch.utils.data.DataLoader(datasets.ImageFolder(
valdir,
transforms.Compose([
transforms.Resize(256),
transforms.CenterCrop(224),
transforms.ToTensor(),
normalize,
])),
batch_size=args.batch_size,
shuffle=False,
num_workers=args.workers,
pin_memory=True)
if args.evaluate:
with torch.no_grad():
validate(val_loader, model, criterion)
return
for epoch in range(args.start_epoch, args.epochs):
adjust_learning_rate(optimizer, epoch)
# train for one epoch
train(train_loader, model, criterion, optimizer, epoch)
with torch.no_grad():
# evaluate on validation set
prec1 = validate(val_loader, model, criterion)
# remember best prec@1 and save checkpoint
is_best = prec1 > best_prec1
best_prec1 = max(prec1, best_prec1)
save_checkpoint(
{
'epoch': epoch + 1,
'arch': args.arch,
'state_dict': model.state_dict(),
'best_prec1': best_prec1,
'optimizer': optimizer.state_dict(),
}, is_best)
# 混淆矩阵
compute_validate_meter(model, 'checkpoint.pth.tar', val_loader)
class RttScriptRunner_TriggerTest:
"""
Class to run any command within the RTT post-processing
'command': command to run (including any arguments)
%TESTNAME is replaced with the jobDisplayName (or test name if not set)
%JOBLOG is replaced with the RTT job log file
%BRANCH is replaces with the release branch
$<ENVVAR> is replaced with the environment variable (if exists)
'logfile': log file, if not given write to RTT logger
"""
def __init__(self, paramDict={}):
from Logger import Logger
self.success = 0
self.error = -1
self.logger = Logger()
self.command = paramDict.get('command', None)
self.logfile = paramDict.get(
'logfile', "test." + self.__class__.__name__ + ".log")
jDescriptor = paramDict.get('JobDescriptor', None)
if jDescriptor:
testname = jDescriptor.jobDisplayName
if testname == '': testname = jDescriptor.name
self.command = self.command.replace("%TESTNAME", testname)
self.command = self.command.replace("%JOBLOG", jDescriptor.log)
self.command = self.command.replace("%BRANCH",
jDescriptor.paths.branch)
for k, v in os.environ.iteritems():
self.command = self.command.replace("$" + k, v)
def run(self):
if not self.command:
self.logger.error("'command' parameter not set")
return self.error
self.logger.debug("ScriptRunner: Executing '%s'" % self.command)
# Run command and get output
import subprocess as sp
proc = sp.Popen(self.command,
shell=True,
stdout=sp.PIPE,
stderr=sp.STDOUT)
stdout = proc.communicate()[0]
# Write output to log file or RTT logger
if self.logfile:
f = open(self.logfile, "w")
f.write(stdout)
f.close()
else:
for l in stdout.splitlines(True):
self.logger.info(l.rstrip('\n'))
if proc.returncode != 0: return self.error
else: return self.success
class MainTab:
"""
This class handles the logic of the main tab
"""
#: The tab specific logger
logger = Logger(Strings.mainTabLoggerName).getLogger()
#: Statusbar labels
statusBarApplicationStatus = None
#: These text will appear in the status bar
statusBarActiveStatuses = []
statusBarInterface = None
statusBarProject = None
___playing = False
@staticmethod
def addApplicationStatus(status):
"""
Add a new status to the status bar by name (ordered).
If no status is present, it will display ``Strings.statusBarReady``
:param status: The new status to add
"""
if status is not None and status != Strings.statusBarReady:
if status not in MainTab.statusBarActiveStatuses:
MainTab.statusBarActiveStatuses.append(status)
# Sort to keep order
MainTab.statusBarApplicationStatus.setText(", ".join(
sorted(MainTab.statusBarActiveStatuses)).title())
else:
MainTab.statusBarApplicationStatus.setText(Strings.statusBarReady)
@staticmethod
def removeApplicationStatus(status):
"""
Remove a status from the status bar.
For statuses with multiple possible values (e.g. ``Sending (X Threads)``
the search will be done using a substring search
:param status: The status to remove
:return:
"""
if status is not None:
if status in MainTab.statusBarActiveStatuses:
MainTab.statusBarActiveStatuses.remove(status)
# For sender threads: search via substring
else:
# If a substring search matches get the element
matchingStatus = None
matchingStatusList = [
value for value in MainTab.statusBarActiveStatuses
if status in value
]
if len(matchingStatusList) > 0:
matchingStatus = matchingStatusList[0]
if matchingStatus is not None:
MainTab.statusBarActiveStatuses.remove(matchingStatus)
if len(MainTab.statusBarActiveStatuses) == 0:
MainTab.statusBarApplicationStatus.setText(
Strings.statusBarReady)
else:
MainTab.statusBarApplicationStatus.setText(", ".join(
sorted(MainTab.statusBarActiveStatuses)).title())
@staticmethod
def setGlobalInterfaceStatus(red=False):
"""
Sets the text of the global interface status in the status bar.
If the global CANData instance is None then the text will read "None".
:param red: Optional; If this is set to True, the text will appear red. Else black.
"""
if red:
MainTab.statusBarInterface.setStyleSheet("QLabel { color : red; }")
else:
MainTab.statusBarInterface.setStyleSheet(
"QLabel { color : black; }")
if Globals.CANData is not None:
MainTab.statusBarInterface.setText("Global interface: " +
Globals.CANData.toString())
else:
MainTab.statusBarInterface.setText("Global interface: None")
@staticmethod
def setProjectStatus(projectName, red=False):
"""
Sets the text of the project status in the status bar.
:param projectName: The text to put as the new project name
:param red: Optional; If this is set to True, the text will appear red. Else black.
"""
if not red:
MainTab.statusBarProject.setStyleSheet("QLabel { color : black; }")
else:
MainTab.statusBarProject.setStyleSheet("QLabel { color : red; }")
MainTab.statusBarProject.setText("Project: " + projectName)
@staticmethod
def populateProjects(keepCurrentIndex=False):
"""
This populates the project ComboBox in the main tab.
:param keepCurrentIndex: If this is set to True, the previously selected index will be re-selected in the end
"""
# Save the index
if keepCurrentIndex:
currentIndex = Globals.ui.comboBoxProjectSet.currentIndex()
projectComboBox = Globals.ui.comboBoxProjectSet
projectComboBox.clear()
projects = Globals.db.getProjects()
for i in range(len(projects)):
projectComboBox.addItem(projects[i].toComboBoxString())
projectComboBox.setItemData(i, projects[i])
# If theres only 1 project then set it as active
# if len(projects) == 1:
# Globals.project = MainTab.setProject()
if keepCurrentIndex:
# Set the remembered index
Globals.ui.comboBoxProjectSet.setCurrentIndex(currentIndex)
@staticmethod
def setProject(wasDeleted=False, setNone=False):
"""
This sets the current project to the currently selected project in the corresponding ComboBox.
Also, the status bar and project specific ComboBoxes and GUI Elements will be updated.
:param wasDeleted: This is set to True if the current selected project was deleted. This causes
``Globals.project`` to become None, too.
:param wasNull: This is set to True, if the project has to be set to None. Default: False
"""
if not wasDeleted and not setNone:
Globals.project = Globals.ui.comboBoxProjectSet.itemData(
Globals.ui.comboBoxProjectSet.currentIndex())
if Globals.project is None:
return
MainTab.setProjectStatus(Globals.project.name)
elif setNone:
Globals.project = None
MainTab.setProjectStatus(Strings.statusBarNoProject, red=True)
else:
Globals.project = None
MainTab.setProjectStatus(Strings.statusBarProjectWasDeleted,
red=True)
MainTab.logger.info(Strings.mainTabLoadingProjectData)
# Update the project data that is managed
Globals.managerTabInstance.getKnownPacketsForCurrentProject()
Globals.managerTabInstance.populatePacketSets()
Globals.managerTabInstance.populateKnownPackets()
Toolbox.Toolbox.toggleDisabledProjectGUIElements()
MainTab.logger.info(
Strings.mainTabProjectSet + " " +
Globals.project.name if Globals.project is not None else "None")
@staticmethod
def loadKernelModules():
"""
Load kernel modules to interact with CAN networks (``can`` and ``vcan``).
"""
cmds = "modprobe can; modprobe vcan"
output, error = Toolbox.Toolbox.runRootshell(cmds)
@staticmethod
def easterEgg(event):
"""
Nothing to see here
:return: fun
"""
if Strings.musicFilePath in Toolbox.Toolbox.mp3Processes:
Toolbox.Toolbox.stopMP3(Strings.musicFilePath)
else:
Toolbox.Toolbox.playMP3(Strings.musicFilePath)
@staticmethod
def addVCANInterface():
"""
Manually add a virtual CAN interface. This uses a syscall to ``ip link``.
If this call succeeds, a new CANDataInstance will be created using :func:`~src.CANData.createCANDataInstance`.
The detected CAN interfaces will be refreshed, too.
"""
vifaceName = "vcan" + str(Globals.ui.spinBoxVCANIndex.value())
cmds = [
"ip link add dev " + vifaceName + " type vcan",
"ip link set up " + vifaceName
]
for cmd in cmds:
output, error = Toolbox.Toolbox.runRootshell(cmd)
if error is not None and error.decode("utf-8") == "":
CANData.createCANDataInstance(vifaceName)
MainTab.logger.info(Strings.mainTabVCANAdded + " " + vifaceName)
else:
MainTab.logger.warn(error.decode("utf-8"))
MainTab.detectCANInterfaces()
@staticmethod
def removeVCANInterface():
"""
This removes the currently selected VCAN interface. This uses a syscall to ``ip link``.
If the removed interface was the current global interface, the global interface will become None.
:return:
"""
vifaceName = "vcan" + str(Globals.ui.spinBoxVCANIndex.value())
wasGlobalInterface = False
if Globals.CANData is not None and vifaceName == Globals.CANData.ifaceName:
wasGlobalInterface = True
# Interface still used or not present -- abort
if not CANData.deleteCANDataInstance(vifaceName):
return
cmd = "ip link delete " + vifaceName
output, error = Toolbox.Toolbox.runRootshell(cmd)
SnifferTab.SnifferTab.removeSniffer(snifferTabName=vifaceName)
MainTab.detectCANInterfaces()
Toolbox.Toolbox.updateInterfaceLabels()
Toolbox.Toolbox.toggleDisabledSenderGUIElements()
if error.decode("utf-8") == "":
MainTab.logger.info(Strings.mainTabVCANRemoved + " " + vifaceName)
if wasGlobalInterface:
Globals.CANData = None
else:
MainTab.logger.warn(error.decode("utf-8"))
MainTab.setGlobalInterfaceStatus(red=True)
@staticmethod
def updateVCANButtons():
"""
Update the text of the buttons to add and remove VCAN interfaces.
"""
currentVal = str(Globals.ui.spinBoxVCANIndex.value())
Globals.ui.buttonVCANAdd.setText("Add vcan" + currentVal)
Globals.ui.buttonVCANRemove.setText("Remove vcan" + currentVal)
@staticmethod
def detectCANInterfaces(updateLabels=True):
"""
Detect CAN and VCAN interfaces available in the system. A syscall to ``/sys/class/net`` is being used for this.
For every detected interface a new CANData instance will be
created using :func:`~src.CANData.createCANDataInstance`.
Also, interface labels and the global interface ComboBox will be updated.
:param updateLabels: Whether to update the interface labels or not
"""
ifaces = sorted(os.listdir("/sys/class/net"))
CANIfaces = []
for iface in ifaces:
if "can" in iface:
CANIfaces.append(iface)
removedInterfaceNames = CANData.rebuildCANDataInstances(CANIfaces)
for removedInterfaceName in removedInterfaceNames:
SnifferTab.SnifferTab.removeSniffer(
snifferTabName=removedInterfaceName)
Toolbox.Toolbox.populateInterfaceComboBox(Globals.ui.comboBoxInterface,
reselectCurrentItem=False)
Toolbox.Toolbox.updateCANDataInstances(
CANData.getGlobalOrFirstInstance())
for CANIface in CANIfaces:
SnifferTab.SnifferTab.addSniffer(CANIface)
if updateLabels:
Toolbox.Toolbox.updateInterfaceLabels()
@staticmethod
def preselectUseBitrateCheckBox():
"""
Preselect the VCAN CheckBox state because we can't use the bitrate along with VCAN interfaces.
"""
currentInterface = Globals.ui.comboBoxInterface.itemData(
Globals.ui.comboBoxInterface.currentIndex())
if currentInterface is None:
return
if currentInterface.VCAN:
Globals.ui.checkBoxMainUseVCAN.setChecked(True)
else:
Globals.ui.checkBoxMainUseVCAN.setChecked(False)
MainTab.VCANCheckboxChanged()
@staticmethod
def applyGlobalInterfaceSettings():
"""
Set the currently selected interface as the global interface.
Also, the bitrate will be updated and GUI elements will be toggled.
The CANData instances of all **inactive** tabs will also be set to the global interface.
"""
selectedInterfaceName = Globals.ui.comboBoxInterface.currentText()
if len(selectedInterfaceName) == 0:
return
selectedBitrate = Globals.ui.spinBoxBitrate.value()
selectedCANData = CANData.CANDataInstances[selectedInterfaceName]
# Set all inactive interfaces to the global interface
Toolbox.Toolbox.updateCANDataInstances(selectedCANData)
Globals.CANData = selectedCANData
MainTab.setGlobalInterfaceStatus()
# It's still active, we set it but won't save the settings though
if selectedCANData.active:
MainTab.logger.info(Strings.activeCANDataWontSave +
selectedCANData.ifaceName)
Globals.ui.spinBoxBitrate.setValue(selectedCANData.bitrate)
Globals.ui.checkBoxMainUseVCAN.setChecked(
QtCore.Qt.Checked if selectedCANData.VCAN else QtCore.Qt.
Unchecked)
return
# It's not active, updated VCAN and the bitrate
else:
selectedCANData.VCAN = Globals.ui.checkBoxMainUseVCAN.isChecked()
if selectedCANData.updateBitrate(selectedBitrate):
Globals.ui.buttonApplyInterface.setStyleSheet(
"background-color: green")
else:
Globals.ui.buttonApplyInterface.setStyleSheet(
"background-color: red")
return
MainTab.logger.info(Strings.mainTabCANConfigUpdated)
# Enable all disabled buttons because an interface was set
Toolbox.Toolbox.toggleDisabledSenderGUIElements()
# Update all labels to the new global interface settings
Toolbox.Toolbox.updateInterfaceLabels()
@staticmethod
def applyLogLevelSetting():
"""
Set the minimum logging level to display messages for.
"""
selectedLevel = Globals.ui.comboBoxLoglevel.currentText()
if selectedLevel == "INFO":
Logger.minLogLevel = logging.INFO
elif selectedLevel == "DEBUG":
Logger.minLogLevel = logging.DEBUG
elif selectedLevel == "WARNING":
Logger.minLogLevel = logging.WARN
elif selectedLevel == "ERROR":
Logger.minLogLevel = logging.ERROR
elif selectedLevel == "CRITICAL":
Logger.minLogLevel = logging.CRITICAL
MainTab.logger.debug(Strings.mainTabLogLevelChanged)
@staticmethod
def setupStatusBar():
"""
Add labels to the status bar and prepare it.
"""
MainTab.statusBarApplicationStatus = QLabel(Strings.statusBarReady,
Globals.ui)
MainTab.statusBarInterface = QLabel(
Strings.statusBarSelectGlobalInterface, Globals.ui)
MainTab.statusBarProject = QLabel(Strings.statusBarNoProject,
Globals.ui)
MainTab.setProjectStatus(Strings.statusBarNoProject)
MainTab.statusBarApplicationStatus.setFrameStyle(QFrame.Panel
| QFrame.Sunken)
MainTab.statusBarInterface.setFrameStyle(QFrame.Panel | QFrame.Sunken)
MainTab.statusBarProject.setFrameStyle(QFrame.Panel | QFrame.Sunken)
MainTab.statusBarApplicationStatus.setStyleSheet(
"QLabel { color : black; }")
MainTab.statusBarInterface.setStyleSheet("QLabel { color : red; }")
MainTab.statusBarProject.setStyleSheet("QLabel { color : red; }")
Globals.ui.statusBar().addPermanentWidget(
MainTab.statusBarApplicationStatus, 1)
Globals.ui.statusBar().addPermanentWidget(MainTab.statusBarInterface,
2)
Globals.ui.statusBar().addPermanentWidget(MainTab.statusBarProject, 3)
@staticmethod
def prepareUI():
"""
1. Setup the status bar
2. Detect CAN interfaces and preselect the VCAN CheckBox
3. Populate project ComboBoxes
4. Add the logo
"""
MainTab.setupStatusBar()
MainTab.detectCANInterfaces()
MainTab.populateProjects()
MainTab.preselectUseBitrateCheckBox()
# Setup the "fork me" ribbon
pixmapLogo = QPixmap(Settings.ICON_PATH)
Globals.ui.labelMainLogo.setPixmap(pixmapLogo)
@staticmethod
def VCANCheckboxChanged():
"""
Clickhandler for the VCAN CheckBox which causes the SpinBox to be toggled.
"""
if Globals.ui.checkBoxMainUseVCAN.isChecked():
Globals.ui.spinBoxBitrate.setEnabled(False)
else:
Globals.ui.spinBoxBitrate.setEnabled(True)
def init_with_three_parameters(self,
n_labels_pass,
train_my_dataset,
sub_zone_value="-1"):
if not sub_zone_value == "-1":
self.sub_lable_name = "H" + str(sub_zone_value)
logger = Logger.set_logger()
ranks = train_my_dataset.get_ranges()
self.n_variables = train_my_dataset.get_ninputs()
self.names = train_my_dataset.get_names()
# int array
self.nlabels_array = [0 for x in range(self.n_variables)]
# bool array
self.varreal_array = [True for x in range(self.n_variables)]
self.database = [[Fuzzy() for y in range(n_labels_pass)]
for x in range(self.n_variables)]
self.database_ini = [[Fuzzy() for y in range(n_labels_pass)]
for x in range(self.n_variables)]
for i in range(0, self.n_variables):
rank = float(abs(float(ranks[i][1]) - float(ranks[i][0])))
self.varreal_array[i] = False
if train_my_dataset.is_nominal(i):
self.nlabels_array[i] = int(rank) + 1
elif train_my_dataset.is_integer(i) and (rank +
1) <= n_labels_pass:
self.nlabels_array[i] = int(rank) + 1
else:
self.nlabels_array[i] = n_labels_pass
self.varreal_array[i] = True
self.database[i] = [Fuzzy() for x in range(self.nlabels_array[i])]
self.database_ini[i] = [
Fuzzy() for x in range(self.nlabels_array[i])
]
if (self.nlabels_array[i] - 1.0) == 0:
mark = 0
else:
mark = float(rank / float(self.nlabels_array[i] - 1.0))
for j in range(0, self.nlabels_array[i]):
self.database[i][j] = Fuzzy()
self.database_ini[i][j] = Fuzzy()
value = float(ranks[i][0]) + mark * (j - 1)
self.database_ini[i][j].x0 = self.database[i][
j].x0 = self.set_value(value, ranks[i][0], ranks[i][1])
value = float(ranks[i][0]) + mark * j
self.database_ini[i][j].x1 = self.database[i][
j].x1 = self.set_value(value, ranks[i][0], ranks[i][1])
value = float(ranks[i][0]) + mark * (j + 1)
self.database_ini[i][j].x3 = self.database[i][
j].x3 = self.set_value(value, ranks[i][0], ranks[i][1])
self.database_ini[i][j].y = self.database[i][j].y = 1.0
if not sub_zone_value == "-1":
self.database[i][
j].name = "L_" + self.sub_lable_name + "_" + str(
j) + "(" + str(self.nlabels_array[i]) + ")"
self.database_ini[i][
j].name = "L_" + self.sub_lable_name + "_" + str(
j) + "(" + str(self.nlabels_array[i]) + ")"
else:
self.database[i][j].name = "L_" + str(j) + "(" + str(
self.nlabels_array[i]) + ")"
self.database_ini[i][j].name = "L_" + str(j) + "(" + str(
self.nlabels_array[i]) + ")"
"""
from myTime import *
# Read configuration file
config = ConfigParser()
config.readfp(open(FULL_MAIN_CONF))
logname = config.get('CS', 'logname') # Full name for the logfile
log_level = config.get('CS', 'log_level') # Level of logging
# If not defined in configuration file, set defaults
if (not logname):
logname = CS + "/log/" + "CS.log"
if (not log_level):
log_level = "DEBUG"
# Enable logging
logger = Logger(logname, log_level, "CS")
logger = logger.getLogger()
form = cgi.FieldStorage()
clientName = form["clientName"].value
host = form["host"].value
machinesString = form["machinesString"].value
logger.info("Execution pdsGetClientListing.py for: " + clientName)
if (form.has_key("glob")):
glob = form["glob"].value
else:
glob = ".*"
if (form.has_key("endDate")):
from GA import GA
from TravelSalesPerson import TravelSalesPerson
from Logger import Logger
import numpy as np
import datetime
log = Logger('C-PR2392-10.log', level='info') # set the name of the log
CROSS_RATE = 0.1 # crossover probability
MUTATE_RATE = 0.02 # mutation probability
POP_SIZE = 10 # the size of population
N_GENERATIONS = 20000 # generations, that is, the number of iterations
DATA_SET = 'PR2392' # the input city data, chosen from the ten data sets
if __name__ == '__main__':
starttime = datetime.datetime.now()
env = TravelSalesPerson(DATA_SET) # get the data of all cities
N_CITIES = env.N_CITIES
ga = GA(DNA_size=N_CITIES,
cross_rate=CROSS_RATE,
mutation_rate=MUTATE_RATE,
pop_size=POP_SIZE) # initialize the population
print(
'Completion of population generation. Population Size: {}. DNA Length of Each Individual: {}'
.format(ga.pop_size, ga.DNA_size))
for generation in range(N_GENERATIONS):
lx, ly = ga.translateDNA(
ga.pop, env.city_position) # get the DNA of the population
fitness, total_distance = ga.get_fitness(
lx,
ly) # get the fitness and the travel distance of the population
config = get_config()
prepare_data(os.path.join(config.data, 'train.txt'),
views=config.num_views,
shuffle=True)
prepare_data(os.path.join(config.data, 'test.txt'),
views=config.num_views,
shuffle=False)
caffe.set_device(0)
caffe.set_mode_gpu()
data_size = get_dataset_size(config, 'trainrotnet')
prepare_solver_file(data_size=data_size)
solver = caffe.get_solver(config.solver)
if not config.test:
LOSS_LOGGER = Logger("{}_loss".format(config.name))
ACC_LOGGER = Logger("{}_acc".format(config.name))
train(config, solver)
config = add_to_config(config, 'weights', highest_model_saved)
config = add_to_config(config, 'test', True)
snapshot = get_highest_model(config)
solver.restore(snapshot)
caffemodel = os.path.splitext(snapshot)[0] + '.caffemodel'
solver.net.copy_from(caffemodel)
solver.test_nets[0].copy_from(caffemodel)
print('Model restored')
eval(config, solver)
def main():
parser = ArgumentParser()
parser.add_argument("-s",
"--spec-path",
dest="specPath",
default="../../SPECS")
parser.add_argument("-x",
"--source-path",
dest="sourcePath",
default="../../stage/SOURCES")
parser.add_argument("-r",
"--rpm-path",
dest="rpmPath",
default="../../stage/RPMS")
parser.add_argument("-i",
"--install-package",
dest="installPackage",
default=False,
action="store_true")
parser.add_argument("-p",
"--publish-RPMS-path",
dest="publishRPMSPath",
default="../../stage/PUBLISHRPMS")
parser.add_argument("-e",
"--publish-XRPMS-path",
dest="publishXRPMSPath",
default="../../stage/PUBLISHXRPMS")
parser.add_argument("-l",
"--log-path",
dest="logPath",
default="../../stage/LOGS")
parser.add_argument("-y", "--log-level", dest="logLevel", default="error")
parser.add_argument("-z",
"--top-dir-path",
dest="topDirPath",
default="/usr/src/photon")
parser.add_argument("-b",
"--build-root-path",
dest="buildRootPath",
default="/mnt")
parser.add_argument("-t",
"--threads",
dest="buildThreads",
default=1,
type=int,
help="Number of working threads")
parser.add_argument("-m",
"--tool-chain-stage",
dest="toolChainStage",
default="None")
parser.add_argument("-c",
"--pullsources-config",
dest="pullsourcesConfig",
default="pullsources.conf")
parser.add_argument("-d", "--dist-tag", dest="dist", default="")
parser.add_argument("-k",
"--input-RPMS-path",
dest="inputRPMSPath",
default=None)
parser.add_argument("-n",
"--build-number",
dest="buildNumber",
default="0000000")
parser.add_argument("-v",
"--release-version",
dest="releaseVersion",
default="NNNnNNN")
parser.add_argument("-u",
"--enable-rpmcheck",
dest="rpmCheck",
default=False,
action="store_true")
parser.add_argument("-a",
"--source-rpm-path",
dest="sourceRpmPath",
default="../../stage/SRPMS")
parser.add_argument("-w",
"--pkginfo-file",
dest="pkgInfoFile",
default="../../stage/pkg_info.json")
parser.add_argument("-g",
"--pkg-build-option-file",
dest="pkgBuildOptionFile",
default="../../common/data/pkg_build_options.json")
parser.add_argument("-q",
"--rpmcheck-stop-on-error",
dest="rpmCheckStopOnError",
default=False,
action="store_true")
parser.add_argument("-bd",
"--publish-build-dependencies",
dest="publishBuildDependencies",
default=False)
parser.add_argument("-pw",
"--package-weights-path",
dest="packageWeightsPath",
default=None)
parser.add_argument("-bt",
"--build-type",
dest="pkgBuildType",
choices=['chroot', 'container'],
default="chroot")
parser.add_argument("-F", "--kat-build", dest="katBuild", default=None)
parser.add_argument("-pj",
"--packages-json-input",
dest="pkgJsonInput",
default=None)
parser.add_argument("PackageName", nargs='?')
options = parser.parse_args()
cmdUtils = CommandUtils()
if not os.path.isdir(options.logPath):
cmdUtils.runCommandInShell("mkdir -p " + options.logPath)
logger = Logger.getLogger("Main", options.logPath, options.logLevel)
errorFlag = False
package = None
pkgInfoJsonFile = options.pkgInfoFile
if not os.path.isdir(options.sourcePath):
logger.error("Given Sources Path is not a directory:" +
options.sourcePath)
errorFlag = True
if not os.path.isdir(options.specPath):
logger.error("Given Specs Path is not a directory:" + options.specPath)
errorFlag = True
if not os.path.isdir(options.publishRPMSPath):
logger.error("Given RPMS Path is not a directory:" +
options.publishRPMSPath)
errorFlag = True
if not os.path.isdir(options.publishXRPMSPath):
logger.error("Given X RPMS Path is not a directory:" +
options.publishXRPMSPath)
errorFlag = True
if not os.path.isdir(options.publishRPMSPath + "/" + platform.machine()):
logger.error("Given RPMS Path is missing " + platform.machine() +
" sub-directory:" + options.publishRPMSPath)
errorFlag = True
if not os.path.isdir(options.publishXRPMSPath + "/" + platform.machine()):
logger.error("Given X RPMS Path is missing " + platform.machine() +
" sub-directory:" + options.publishXRPMSPath)
errorFlag = True
if not os.path.isdir(options.publishRPMSPath + "/noarch"):
logger.error("Given RPMS Path is missing noarch sub-directory:" +
options.publishRPMSPath)
errorFlag = True
if not os.path.isdir(options.publishXRPMSPath + "/noarch"):
logger.error("Given X RPMS Path is missing noarch sub-directory:" +
options.publishXRPMSPath)
errorFlag = True
if options.inputRPMSPath is not None and not os.path.isdir(
options.inputRPMSPath):
logger.error("Given input RPMS Path is not a directory:" +
options.inputRPMSPath)
errorFlag = True
if options.packageWeightsPath is not None and not os.path.isfile(
options.packageWeightsPath):
logger.error("Given input Weights file is not a file:" +
options.packageWeightsPath)
errorFlag = True
if options.pkgJsonInput is not None and not os.path.isfile(
options.pkgJsonInput):
logger.error("Given input packages file is not a file:" +
options.pkgJsonInput)
errorFlag = True
if options.installPackage:
if not options.PackageName:
logger.error("Please provide package name")
errorFlag = True
else:
package = options.PackageName
if errorFlag:
logger.error(
"Found some errors. Please fix input options and re-run it.")
return False
if not os.path.isdir(options.rpmPath):
cmdUtils.runCommandInShell("mkdir -p " + options.rpmPath + "/" +
platform.machine())
cmdUtils.runCommandInShell("mkdir -p " + options.rpmPath + "/noarch")
if not os.path.isdir(options.sourceRpmPath):
cmdUtils.runCommandInShell("mkdir -p " + options.sourceRpmPath)
if not os.path.isdir(options.buildRootPath):
cmdUtils.runCommandInShell("mkdir -p " + options.buildRootPath)
logger.debug("Source Path :" + options.sourcePath)
logger.debug("Spec Path :" + options.specPath)
logger.debug("Rpm Path :" + options.rpmPath)
logger.debug("Log Path :" + options.logPath)
logger.debug("Log Level :" + options.logLevel)
logger.debug("Top Dir Path :" + options.topDirPath)
logger.debug("Publish RPMS Path :" + options.publishRPMSPath)
logger.debug("Publish X RPMS Path :" + options.publishXRPMSPath)
if options.installPackage:
logger.debug("Package to build:" + package)
get_packages_with_build_options(options.pkgBuildOptionFile)
try:
constants.setSpecPath(options.specPath)
constants.setSourcePath(options.sourcePath)
constants.setRpmPath(options.rpmPath)
constants.setSourceRpmPath(options.sourceRpmPath)
constants.setTopDirPath(options.topDirPath)
constants.setLogPath(options.logPath)
constants.setLogLevel(options.logLevel)
constants.setDist(options.dist)
constants.setBuildNumber(options.buildNumber)
constants.setReleaseVersion(options.releaseVersion)
constants.setPrevPublishRPMRepo(options.publishRPMSPath)
constants.setPrevPublishXRPMRepo(options.publishXRPMSPath)
constants.setBuildRootPath(options.buildRootPath)
constants.setPullSourcesURL(get_baseurl(options.pullsourcesConfig))
constants.setInputRPMSPath(options.inputRPMSPath)
constants.setRPMCheck(options.rpmCheck)
constants.setRpmCheckStopOnError(options.rpmCheckStopOnError)
constants.setPublishBuildDependencies(options.publishBuildDependencies)
constants.setPackageWeightsPath(options.packageWeightsPath)
constants.setKatBuild(options.katBuild)
constants.initialize()
# parse SPECS folder
SPECS()
if options.toolChainStage == "stage1":
pkgManager = PackageManager()
pkgManager.buildToolChain()
elif options.toolChainStage == "stage2":
pkgManager = PackageManager()
pkgManager.buildToolChainPackages(options.buildThreads)
elif options.installPackage:
buildSpecifiedPackages([package], options.buildThreads,
options.pkgBuildType)
elif options.pkgJsonInput:
buildPackagesInJson(options.pkgJsonInput, options.buildThreads,
options.pkgBuildType, pkgInfoJsonFile, logger)
else:
buildPackagesForAllSpecs(options.buildThreads,
options.pkgBuildType, pkgInfoJsonFile,
logger)
except Exception as e:
logger.error("Caught an exception")
logger.error(str(e))
# print stacktrace
traceback.print_exc()
sys.exit(1)
sys.exit(0)
def modus_ponens(A,B): if A == B[1:len(A)+1]: Logger(B[len(A)+2:len(B)-1], 'Modus ponens for '+A+' and '+B) return B[len(A)+2:len(B)-1]
RCK = 0x20
CTR = 0x00
SDA = 0x1c
RDA = 0x12
CLK = 0x18
if bmi + bmo != 1:
raise ValueError("must specify one of 'bmi' or 'bmo' at cmd line")
for x in (RCK, CTR, SDA, RDA, CLK):
assert 0 <= x <= 255
########################################################
print "connecting to ccu prog %s:%i" % (fechost, fecport)
ccu = SimpleSocket(fechost, fecport)
log = Logger()
print
print 'not sending cratereset, reset, piareset all as old script did'
tag="FEC %d ring 0x%i :"%(fecslot,fecring)
print 'tag is', tag
fecringfound=False
fecfound=False
status=""
for l in ccu.send("mapccu").readlines():
print l
if l.strip().startswith("FEC %d"%(fecslot)):
fecfound=True
if l.strip().startswith(tag):
fecringfound=True
status=l.strip().split(":")[1].strip()
def A3(F,G):
P = IMPLICATION(IMPLICATION(NOT(G),NOT(F)),IMPLICATION(IMPLICATION(NOT(G),F),G))
Logger(P,'A3 for '+F+' and '+G)
return '('+'('+'('+NEG+G+')'+IMP+'('+NEG+F+')'+')'+IMP+'('+'('+'('+NEG+G+')'+IMP+F+')'+IMP+G+')'+')'
def A2(F,G,H): P = IMPLICATION(IMPLICATION(F,IMPLICATION(G,H)),IMPLICATION(IMPLICATION(F,G),IMPLICATION(F,H))) Logger(P,'A2 for '+F+' and '+G+' and '+H) return P
class ACL(QObject):
updateError = pyqtSignal(name='updateError')
update = pyqtSignal()
downloadActiveUpdate = pyqtSignal()
@pyqtProperty(str, notify=update)
def hash(self):
self.mutex.lock()
h = self._hash
self.mutex.unlock()
return h
@pyqtProperty(int, notify=update)
def numRecords(self):
self.mutex.lock()
n = self._numRecords
self.mutex.unlock()
return n
@pyqtProperty(int, notify=update)
def numActiveRecords(self):
self.mutex.lock()
n = self._numActiveRecords
self.mutex.unlock()
return n
@pyqtProperty(str, notify=update)
def date(self):
self.mutex.lock()
d = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(self._date))
self.mutex.unlock()
return d
@pyqtProperty(str, notify=update)
def status(self):
self.mutex.lock()
s = self._status
self.mutex.unlock()
return s
@pyqtProperty(bool, notify=downloadActiveUpdate)
def downloadActive(self):
self.mutex.lock()
isdl = self._downloadActive
self.mutex.unlock()
return isdl
@downloadActive.setter
def downloadActive(self, value):
self.mutex.lock()
self._downloadActive = value
self.mutex.unlock()
self.downloadActiveUpdate.emit()
def __init__(self,
loglevel='WARNING',
netWorker=None,
url='',
cacheFile=None):
QObject.__init__(self)
self.logger = Logger(name='ratt.acl')
self.logger.setLogLevelStr(loglevel)
self.debug = self.logger.isDebug()
if netWorker is None:
self.logger.error('must set netWorker')
exit(-1)
self.netWorker = netWorker
self.url = url
self.cacheFile = cacheFile
self.mutex = QMutex()
self._acl = json.loads('[]')
self._hash = ''
self._numRecords = 0
self._numActiveRecords = 0
self._downloadActive = False
self._date = int(time.time())
self._status = 'initialized'
self.loadFile(self.cacheFile)
def loadFile(self, filename=None):
self.logger.info('loading ACL file %s' % filename)
if filename is not None:
f = QFile(filename)
if not f.open(QIODevice.ReadOnly | QIODevice.Text):
self.logger.error('error opening ACL file %s for read' %
filename)
return False
bytes = f.readAll()
if bytes.isEmpty():
self.logger.error('unabled to read from ACL file %s' %
filename)
f.close()
return False
f.close()
info = QFileInfo(filename)
modified = int(info.lastModified().toMSecsSinceEpoch() / 1000)
return self.parseJSON(doc=str(bytes),
save=False,
date=modified,
status='loaded_from_file')
return False
def saveFile(self, filename=None):
self.logger.info('saving ACL file %s' % filename)
if filename is not None:
self.mutex.lock()
doc = json.dumps(self._acl)
self.mutex.unlock()
f = QFile(filename)
if not f.open(QIODevice.WriteOnly):
self.logger.error('error opening ACL file %s for write' %
filename)
return False
if f.write(str(doc)) == -1:
f.close()
self.logger.error('unabled to write to ACL file %s' % filename)
return False
f.close()
return True
return False
@pyqtSlot()
def download(self):
if not self.downloadActive:
self.logger.info('downloading ACL from ' + self.url)
self.reply = self.netWorker.get(url=QUrl(self.url))
self.reply.finished.connect(self.slotDownloadFinished)
self.downloadActive = True
return True
else:
self.logger.warning('already busy downloading ACL')
return False
def slotDownloadFinished(self):
self.logger.info('ACL download finished')
error = self.reply.error()
if error == QNetworkReply.NoError:
self.logger.debug('no error, parsing response')
self.parseJSON(doc=str(self.reply.readAll()),
save=True,
status='downloaded')
else:
self.logger.error('NetWorker response error: %s (%s)' %
(error, self.reply.errorString()))
self.reply.deleteLater()
self.downloadActive = False
def search(self, hash):
self.mutex.lock()
found_record = []
for record in self._acl:
if record['tagid'] == hash:
found_record = record
break
self.mutex.unlock()
return found_record
def countActive(self, j):
active = 0
for record in j:
if record['allowed'] == 'allowed':
active = active + 1
return active
def calcHash(self, j):
m = hashlib.sha224()
m.update(str(j).encode())
hash = m.hexdigest()
return hash
def parseJSON(self, doc, save=False, date=None, status=''):
try:
parsed = json.loads(doc)
count = len(parsed)
active = self.countActive(parsed)
hash = self.calcHash(parsed)
self.logger.debug('parseJSON hash=%s stored hash=%s' %
(hash, self._hash))
if hash != self._hash:
self.mutex.lock()
self._acl = parsed
self._numRecords = count
self._numActiveRecords = active
self._hash = hash
if date is None:
self._date = int(time.time())
else:
self._date = date
self._status = status
self.mutex.unlock()
self.logger.info(
'updated ACL with %d entries, %d active, hash=%s' %
(self._numRecords, self._numActiveRecords, self._hash))
self.update.emit()
if save:
self.saveFile(self.cacheFile)
return True
else:
self.mutex.lock()
if date is None:
self._date = int(time.time())
self._status = 'same_hash'
self.mutex.unlock()
self.update.emit()
self.logger.error('no ACL update because same hash=%s' %
(self._hash))
except:
self.logger.exception('json parse error')
self.updateError.emit()
return False
from Logger import Logger
from utils.FileUtils import FileUtils
def get_root_path():
"""
get project root path
:return: root path
"""
fileRoot = getattr(sys.modules['__main__'], '__file__')
return os.path.abspath(os.path.dirname(fileRoot))
# load banner #
banner = os.path.join(get_root_path(), 'banner.txt')
Logger.info('load banner from %s', banner)
if FileUtils.exists(banner):
file = open(file=banner)
print(file.read())
file.close()
else:
Logger.info('banner not exists, skip it')
# load configuration #
Logger.info('start load default configuration file')
config = os.path.join(get_root_path(), 'configuration', 'sql-view.yaml')
Logger.info('load configuration from %s', config)
try:
file = open(config, 'r')
Configuration = yaml.load(file, Loader=yaml.FullLoader)
except Exception as ex:
reload(sys)
sys.setdefaultencoding('utf8')
#以上三行代码放处理中文的代码前面
#################################################################################################
#logging.basicConfig(level=logging.DEBUG,format='%(asctime)s %(filename)s[line:%(lineno)d] %(levelname)s %(message)s',datefmt='%a, %d %b %Y %H:%M:%S', filename='myapp.log',filemode='w')
#定义一个StreamHandler，将INFO级别或更高的日志信息打印到标准错误，并将其添加到当前的日志处理对象#
#console = logging.StreamHandler()
#console.setLevel(logging.INFO)
#formatter = logging.Formatter('%(name)-12s: %(levelname)-8s %(message)s')
#console.setFormatter(formatter)
#logging.getLogger('').addHandler(console)
#################################################################################################
#日志工具
logger = Logger('log/all.log', level='debug').logger
wechat = {}
wechat['window_x'] = 600 #微信聊天窗口打字位置 x坐标
wechat['window_y'] = 600 #微信聊天窗口打字位置 y坐标
wechat['content_x'] = 680 #微信URL x坐标
wechat['content_y'] = 450 #微信URL y坐标
wechat['window_web_x'] = 816 #微信浏览器关闭位置 x坐标
wechat['window_web_y'] = 12 #微信聊天窗口打字位置 y坐标
URL = "collect_wechat_run_usersub_pc.php?web=2"
URL_IP = "shell.myzaker.com"
ANALYSE_USE_TIME_ARR = [] #模拟记录每次的处理时间（传入1：表示有处理，传入3：表示没处理）
#!/usr/bin/python
import json
import settings
import pymongo
from bson import ObjectId
from Logger import Logger
from database import MongoDB
DATABASE = settings.database
COLLECTION = settings.collection
# Create an instance of logger class
logger = Logger()
class Media(dict):
"""
A Media model that creates movie/show type document on mongodb
"""
def __init__(self, dict):
""" Class instructor """
pass
def store(self):
""" A function in this class to store a media, a movie or a show into the database """
client = MongoDB.setupConnection()
try:
db = client[DATABASE]
collection = db[COLLECTION]
return collection.insert_one(self).inserted_id
except Exception as error:
class PullFTP(object):
def __init__(self, source, logger=None, sleeping=False) :
# General Attributes
self.source = source
self.sleeping = sleeping
self.connected = False
self.ftp = None
self.sftp = None
self.file = None
self.ls = {}
self.lsold = {}
self.lspath = ''
self.pulllst = []
self.originalDir = ''
self.destDir = ''
if logger is None:
self.logger = Logger(PXPaths.LOG + 'rx_' + source.name + '.log', 'INFO', 'RX' + name) # Enable logging
self.logger = self.logger.getLogger()
else:
self.logger = logger
if self.source.delete and self.sleeping :
return
if self.source.protocol == 'ftp':
self.ftp = self.ftpConnect()
if self.ftp != None :
self.chdir = self.ftp.cwd
self.delete = self.ftp.delete
self.fget = self.ftp_get
self.lsdir = self.ftp_do_ls
self.quit = self.ftp.quit
if self.source.protocol == 'sftp':
self.sftp = self.sftpConnect()
if self.sftp != None :
self.chdir = self.sftp.chdir
self.delete = self.sftp.remove
self.fget = self.sftp.get
self.lsdir = self.sftp_do_ls
self.quit = self.sftp_quit
# callback to strip and save the output from ls of directory
def callback_line(self,block):
# strip ls line to its most important info
fil,desc = self.ls_line_stripper(block)
# keep only if we are interested in that file
for filepattern in self.pulllst :
self.logger.debug("fil = (%s)" % fil)
if re.compile(filepattern).match(fil):
self.logger.debug("fil = OK")
self.ls[fil] = desc
# going to a certain directory
def cd(self, path):
timex = AlarmFTP('FTP change working directory timeout')
try :
# gives 10 seconds to get there
timex.alarm(10)
self.chdir(self.originalDir)
self.chdir(path)
timex.cancel()
return True
except :
timex.cancel()
(type, value, tb) = sys.exc_info()
self.logger.warning("Could not cd to directory %s" % path )
self.logger.warning(" Type: %s, Value: %s" % (type ,value))
return False
# close connection...
def close(self):
self.connected = False
# connection did not work
if self.ftp == None and self.sftp == None : return
timex = AlarmFTP(self.source.protocol + ' connection timeout')
try :
# gives 10 seconds to close the connection
timex.alarm(10)
self.quit()
timex.cancel()
except :
timex.cancel()
(type, value, tb) = sys.exc_info()
self.logger.warning("Could not close connection")
self.logger.warning(" Type: %s, Value: %s" % (type ,value))
# find differences between current ls and last ls
# only the newer or modified files will be kept...
def differ(self):
# get new list and description
new_lst = self.ls.keys()
new_desc = self.ls
new_lst.sort()
# get old list and description
self.load_ls_file(self.lspath)
old_lst = self.lsold.keys()
old_desc = self.lsold
old_lst.sort()
# compare
filelst = []
desclst = {}
for f in new_lst :
# keep a newer entry
if not f in old_lst :
filelst.append(f)
desclst[f] = new_desc[f]
continue
# keep a modified entry
if new_desc[f] != old_desc[f] :
filelst.append(f)
desclst[f] = new_desc[f]
continue
return filelst,desclst
# check for pattern matching in directory name
def dirPattern(self,path) :
"""
Replace pattern in directory...
"""
ndestDir = ''
DD = path.split("/")
for ddword in DD[1:] :
ndestDir += '/'
if ddword == "" : continue
nddword = ""
DW = ddword.split("$")
for dwword in DW :
nddword += self.matchPattern(dwword,dwword)
ndestDir += nddword
return ndestDir
# open connection... ftp
def ftpConnect(self):
timex = AlarmFTP('FTP connection timeout')
try:
# gives 30 seconds to open the connection
timex.alarm(30)
ftp = ftplib.FTP(self.source.host, self.source.user, self.source.passwd)
if self.source.ftp_mode == 'active':
ftp.set_pasv(False)
else:
ftp.set_pasv(True)
try : self.originalDir = ftp.pwd()
except:
(type, value, tb) = sys.exc_info()
self.logger.warning("Unable to ftp.pwd (Type: %s, Value: %s)" % (type ,value))
timex.cancel()
self.connected = True
return ftp
except FtpTimeoutException :
timex.cancel()
self.logger.warning("FTP connection timed out after 30 seconds..." )
except:
timex.cancel()
(type, value, tb) = sys.exc_info()
self.logger.error("Unable to connect to %s (user:%s). Type: %s, Value: %s" % (self.source.host, self.source.user, type ,value))
# ftp do an ls in the current directory, write in file path
def ftp_do_ls(self):
self.ls = {}
timex = AlarmFTP('FTP retrieving list')
timex.alarm(self.source.timeout_get)
try :
ls=self.ftp.retrlines('LIST',self.callback_line )
timex.cancel()
return True
except FtpTimeoutException :
timex.cancel()
self.logger.warning("FTP doing ls timed out after %s seconds..." % self.source.timeout_get )
except :
timex.cancel()
(type, value, tb) = sys.exc_info()
self.logger.warning("Could not ls directory %s" % self.destDir)
self.logger.warning(" Type: %s, Value: %s" % (type ,value))
return False
# ftp retrieve a file
def ftp_get(self, remote_file, local_file):
file=open(local_file,'wb')
self.ftp.retrbinary('RETR ' + remote_file, file.write )
file.close()
# pulling files and returning the list
def get(self):
# if pull is sleeping and we delete files... nothing to do
# if we don't delete files, we will keep the directory state
if self.source.delete and self.sleeping : return
# log that we are waking up
if not self.sleeping :
self.logger.info("pull %s is waking up" % self.source.name )
# getting our alarm ready
timex = AlarmFTP('FTP timeout')
# files list to return
files_pulled = []
# connection did not work
if self.ftp == None and self.sftp == None : return files_pulled
# loop on all directories where there are pulls to do
for lst in self.source.pulls :
self.destDir = lst[0]
self.pulllst = lst[1:]
pdir = self.dirPattern(self.destDir)
if pdir != '' : self.destDir = pdir
self.destDir = self.destDir[1:]
# cd to that directory
self.logger.debug(" cd %s" % self.destDir)
ok = self.cd(self.destDir)
if not ok : continue
# create ls filename for that directory
pdir = lst[0]
pdir = pdir.replace('${','')
pdir = pdir.replace('}','')
self.lspath = PXPaths.RXQ + self.source.name + '/.ls' + pdir.replace('/','_')
# ls that directory
ok = self.lsdir()
if not ok : continue
# if we are sleeping and we are here it is because
# this pull is retrieving difference between directory content
# so write the directory content without retrieving files
if self.sleeping :
ok = self.write_ls_file(self.lspath)
continue
# get the file list from the ls
filelst = self.ls.keys()
desclst = self.ls
# if we dont delete, get file list from difference in ls
if not self.source.delete :
filelst,desclst = self.differ()
if len(filelst) == 0 :
ok = self.write_ls_file(self.lspath)
continue
# get remote file time for all files
timelst,filelst,desclst = self.remote_time_sort(filelst,desclst)
# retrieve the files
files_notretrieved = []
for idx,remote_file in enumerate(filelst) :
timex.alarm(self.source.timeout_get)
local_file = self.local_filename(remote_file,desclst,timelst)
try :
ok = self.retrieve(remote_file, local_file)
if ok :
if self.source.delete : self.rm(remote_file)
files_pulled.append(local_file)
# setting access,modify time to remote time
ftime = time.mktime(timelst[remote_file])
os.utime(local_file,(ftime,ftime) )
else :
files_notretrieved.extend(filelst[idx:])
self.logger.warning("problem when retrieving %s " % remote_file )
break
# if batch is reached stop there
if len(files_pulled) == self.source.batch :
if idx != len(filelst)-1 : files_notretrieved.extend(filelst[idx+1:])
break
timex.cancel()
except FtpTimeoutException :
timex.cancel()
files_notretrieved.extend(filelst[idx:])
self.logger.warning("FTP timed out retrieving %s " % remote_file )
break
except :
timex.cancel()
files_notretrieved.extend(filelst[idx:])
(type, value, tb) = sys.exc_info()
self.logger.error("Unable write remote file %s in local file %s" % (remote_file,local_file))
self.logger.error(" Type: %s, Value: %s" % (type ,value))
break
# files not retrieved are removed from the file list
# this allow pull to recover from error on next pass
for f in files_notretrieved :
del self.ls[f]
# save ls file
ok = self.write_ls_file(self.lspath)
# if reached the batch limit : break here
if len(files_pulled) == self.source.batch :
self.logger.warning("break retrieving... batch reached")
break
# if we had a problem break here
if len(files_notretrieved) > 0 :
self.logger.warning("break retrieving... because of error")
break
return files_pulled
# parse a file containing an ls and return the file entries
def load_ls_file(self,path):
self.lsold = {}
if not os.path.isfile(path) : return True
try :
# open/read..
file=open(path,'rb')
lines=file.readlines()
file.close()
# get filenames
for line in lines :
parts = line.split()
fil = parts[-1]
self.lsold[fil] = line
return True
except:
self.logger.error("Unable to parse files from %s" % path )
return False
# create local filename
def local_filename(self,filename,desclst,timelst):
# create local filename with date/time on host... YYYYMMDDHHMM_filename
if self.source.pull_prefix == 'HDATETIME' :
ftime = timelst[filename]
datetimestr = time.strftime("%Y%m%d%H%M",ftime)
local_file = PXPaths.RXQ + self.source.name + '/' + datetimestr + '_' + filename
return local_file
# create local filename with config defined prefix ... prefix_filename
if self.source.pull_prefix != '' :
local_file = PXPaths.RXQ + self.source.name + '/' + self.source.prefix + '_' + filename
return local_file
# create local filename with same name as remote file (default)
local_file = PXPaths.RXQ + self.source.name + '/' + filename
return local_file
# ls line stripper
def ls_line_stripper(self,iline):
oline = iline
oline = oline.strip()
oline = oline.replace('\t',' ')
opart1 = oline.split(' ')
opart2 = []
for p in opart1 :
if p == '' : continue
opart2.append(p)
fil = opart2[-1]
desc = opart2[0] + ' ' + ' '.join(opart2[-5:]) + '\n'
return fil,desc
# replace a matching pattern by its value in the directory name
def matchPattern(self,keywd,defval) :
"""
Matching keyword with different patterns
"""
if keywd[:6] == "{YYYY}" :
return time.strftime("%Y", time.gmtime()) + keywd[6:]
if keywd[:9] == "{YYYY-1D}" :
epoch = time.mktime(time.gmtime()) - 24*60*60
return time.strftime("%Y", time.localtime(epoch) ) + keywd[9:]
if keywd[:4] == "{MM}" :
return time.strftime("%m", time.gmtime()) + keywd[4:]
if keywd[:7] == "{MM-1D}" :
epoch = time.mktime(time.gmtime()) - 24*60*60
return time.strftime("%m", time.localtime(epoch) ) + keywd[7:]
if keywd[:5] == "{JJJ}" :
return time.strftime("%j", time.gmtime()) + keywd[5:]
if keywd[:8] == "{JJJ-1D}" :
epoch = time.mktime(time.gmtime()) - 24*60*60
return time.strftime("%j", time.localtime(epoch) ) + keywd[8:]
if keywd[:10] == "{YYYYMMDD}" :
return time.strftime("%Y%m%d", time.gmtime()) + keywd[10:]
if keywd[:13] == "{YYYYMMDD-1D}" :
epoch = time.mktime(time.gmtime()) - 24*60*60
return time.strftime("%Y%m%d", time.localtime(epoch) ) + keywd[13:]
if keywd[:13] == "{YYYYMMDD-2D}" :
epoch = time.mktime(time.gmtime()) - 48*60*60
return time.strftime("%Y%m%d", time.localtime(epoch) ) + keywd[13:]
if keywd[:13] == "{YYYYMMDD-3D}" :
epoch = time.mktime(time.gmtime()) - 72*60*60
return time.strftime("%Y%m%d", time.localtime(epoch) ) + keywd[13:]
if keywd[:13] == "{YYYYMMDD-4D}" :
epoch = time.mktime(time.gmtime()) - 96*60*60
return time.strftime("%Y%m%d", time.localtime(epoch) ) + keywd[13:]
if keywd[:13] == "{YYYYMMDD-5D}" :
epoch = time.mktime(time.gmtime()) - 120*60*60
return time.strftime("%Y%m%d", time.localtime(epoch) ) + keywd[13:]
return defval
# get remote file time
def remote_file_time(self,filename,desc):
ftime = time.localtime()
try :
line = desc[filename]
line = line.strip()
parts = line.split()
datestr=' '.join(parts[-4:-1])
fmt4 = "%b"
fmt3 = "%d"
fmt2 = "%Y"
if len(parts[-2]) == 5 :
fmt2 = "%H:%M %Y"
Y = time.strftime("%Y",time.gmtime())
datestr = datestr + ' ' + Y
if parts[-3][0].isalpha() : fmt3 = "%b"
if parts[-4][0].isdigit() : fmt4 = "%d"
format = fmt4 + " " + fmt3 + " " + fmt2
ftime = time.strptime(datestr,format)
except : pass
return ftime
# remote time sort of file to pull
def remote_time_sort(self,ifilelst,idesclst):
timelst = [ self.remote_file_time(rfile,idesclst) for idx,rfile in enumerate(ifilelst) ]
templst = [ (time.mktime(timelst[idx]),timelst[idx],rfile,idesclst[rfile]) for idx,rfile in enumerate(ifilelst) ]
templst.sort()
filelst = []
desclst = {}
timelst = {}
for tup in templst :
epoch, rtime, rfile, rdesc = tup
filelst.append(rfile)
desclst[rfile] = rdesc
timelst[rfile] = rtime
return timelst,filelst,desclst
# retrieve a file
def retrieve(self, remote_file, local_file):
try :
self.fget( remote_file, local_file)
return True
except :
os.unlink(local_file)
(type, value, tb) = sys.exc_info()
self.logger.error("Unable write remote file %s in local file %s. Type: %s, Value: %s" % \
(remote_file,local_file,type,value))
return False
# some systems do not permit deletion... so pass exception on that
def rm(self, path):
try :
self.delete(path)
except :
(type, value, tb) = sys.exc_info()
self.logger.warning("Could not delete %s" % path )
self.logger.warning("Type: %s, Value: %s" % (type ,value))
# open connection... sftp
def sftpConnect(self):
timex = AlarmFTP('SFTP connection timeout')
# gives 30 seconds to open the connection
try:
timex.alarm(30)
self.t = None
if self.source.port == None :
self.t = paramiko.Transport(self.source.host)
else:
t_args = (self.source.host,self.source.port)
self.t = paramiko.Transport(t_args)
if self.source.ssh_keyfile != None :
#TODO, implement password to use to decrypt the key file, if it's encrypted
key=DSSKey.from_private_key_file(self.source.ssh_keyfile,password=None)
self.t.connect(username=self.source.user,pkey=key)
else:
self.t.connect(username=self.source.user,password=self.source.passwd)
self.sftp = paramiko.SFTP.from_transport(self.t)
# WORKAROUND without going to '.' originalDir was None
self.sftp.chdir('.')
self.originalDir = self.sftp.getcwd()
timex.cancel()
self.connected = True
return self.sftp
except FtpTimeoutException :
timex.cancel()
try : self.sftp.close()
except : pass
try : self.t.close()
except : pass
self.logger.error("SFTP connection timed out after %d seconds... retrying" % 30 )
except:
timex.cancel()
(type, value, tb) = sys.exc_info()
self.logger.error("Unable to connect to %s (user:%s). Type: %s, Value: %s" % (self.source.host, self.source.user, type ,value))
try : self.sftp.close()
except : pass
try : self.t.close()
except : pass
# do an ls in the current directory, write in file path
def sftp_do_ls(self):
self.ls = {}
timex = AlarmFTP('SFTP retrieving list')
timex.alarm(self.source.timeout_get)
try :
dir_attr = self.sftp.listdir_attr()
for index in range(len(dir_attr)):
attr = dir_attr[index]
line = attr.__str__()
self.callback_line(line)
timex.cancel()
return True
except FtpTimeoutException :
timex.cancel()
self.logger.warning("SFTP doing ls timed out after %s seconds..." % self.source.timeout_get )
except :
timex.cancel()
(type, value, tb) = sys.exc_info()
self.logger.warning("Could not ls directory %s" % self.destDir)
self.logger.warning(" Type: %s, Value: %s" % (type ,value))
return False
# sftp quit = close connection...
def sftp_quit(self):
self.sftp.close()
self.t.close()
# write ls file
def write_ls_file(self,path):
filelst = self.ls.keys()
desclst = self.ls
filelst.sort()
try :
# open/write..
file=open(path,'wb')
for f in filelst :
file.write(desclst[f])
file.close()
return True
except:
self.logger.error("Unable to write ls to file %s" % path )
return False
def prt(self):
Logger.dbg('x : {0} - y : {1}'.format(self.x, self.y))
dryRun = False
try:
if sys.argv.index('--dryrun') > 0:
dryRun = True
except ValueError:
pass
def timestamp():
ts = time.time()
return datetime.datetime.fromtimestamp(ts).strftime('%Y-%m-%d %H:%M:%S')
bot = Poloniex(config.get("API", "apikey"), config.get("API", "secret"))
log = Logger()
#total lended global variable
totalLended = {}
def refreshTotalLended():
global totalLended, rateLended
cryptoLended = bot.returnActiveLoans()
totalLended = {}
rateLended = {}
cryptoLendedSum = Decimal(0)
cryptoLendedRate = Decimal(0)
for item in cryptoLended["provided"]:
class Trainer(object):
def __init__(self, cfg, ckpt_path=None):
self.cfg = cfg
random.seed(self.cfg.SEED)
torch.manual_seed(self.cfg.SEED)
torch.cuda.manual_seed(self.cfg.SEED)
dt_now = dt.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
self.experiment_name = self.cfg.PROJECT_NAME + '_' + dt_now
self.experiment_path = os.path.join(self.cfg.EXPERIMENTS_PATH,
self.experiment_name)
self.make_folders()
self.logger = Logger(self.experiment_path,
to_file=True,
to_tensorboard=True)
#self.env = AtariEnv(self.cfg)
self.env = CartPoleEnv(self.cfg)
if self.cfg.USE_GPU:
self.gpu_id = 0
self.device = torch.device('cuda', self.gpu_id)
else:
self.device = torch.device('cpu')
#self.target_model = DQN(self.cfg).to(self.device)
self.target_model = DQNMLP(self.cfg).to(self.device)
self.target_model.eval()
#self.model = DQN(self.cfg).to(self.device)
self.model = DQNMLP(self.cfg).to(self.device)
self.model.train()
self.logger.log_config(self.cfg)
self.logger.log_pytorch_model(self.model)
self.ckpt_path = os.path.join(
self.experiment_path, 'ckpt',
self.target_model.model_name + '.weights')
if self.cfg.OPTIM == 'adam':
self.optimizer = optim.Adam(filter(lambda p: p.requires_grad,
self.model.parameters()),
lr=self.cfg.LEARNING_RATE)
#self.optimizer = optim.Adam(parameters, lr=self.cfg.LEARNING_RATE)
elif self.cfg.OPTIM == 'rms-prop':
self.optimizer = optim.RMSprop(filter(lambda p: p.requires_grad,
self.model.parameters()),
lr=self.cfg.LEARNING_RATE)
#self.optimizer = optim.RMSprop(parameters,
# lr=self.cfg.LEARNING_RATE)
elif self.cfg.OPTIM == 'sgd':
self.optimizer = optim.SGD(filter(lambda p: p.requires_grad,
self.model.parameters()),
lr=self.cfg.LEARNING_RATE)
if ckpt_path is not None:
self.model.load_state_dict(torch.load(ckpt_path))
def train(self):
"""
Procedure to run the training procedure
"""
self.target_model.load_state_dict(self.model.state_dict())
experience_replay = ExperienceReplay(self.cfg)
epsilon = self.cfg.EPS_START
drop_epsilon = (self.cfg.EPS_START -
self.cfg.EPS_END) / self.cfg.EPS_DECAY
global_step = 0
epsilon_step = 0
for e in range(self.cfg.MAX_EPISODES):
self.env.reset()
state = self.env.get_state()
while not self.env.done:
global_step += 1
if self.cfg.RENDER:
self.env.render()
if global_step > self.cfg.TRAIN_START:
epsilon_step += 1
epsilon = self.cfg.EPS_END + (self.cfg.EPS_START - self.cfg.EPS_END) * \
math.exp(-1. * epsilon_step / self.cfg.EPS_DECAY)
self.logger.log_value('epsilon',
global_step,
epsilon,
print_value=False,
to_file=False)
if random.random() < epsilon:
# epsilon-greedy exploration
action = torch.tensor(
[[random.randrange(self.cfg.NUM_ACTIONS)]],
dtype=torch.long)
else:
with torch.no_grad():
q_pred = self.model(state.to(self.device))
_, action = q_pred.max(1)
action = action.view(1, 1).cpu()
reward = self.env.step(action.item())
r = np.clip(reward, -1.0, 1.0)
r = torch.tensor([reward])
if not self.env.done:
next_state = self.env.get_state()
else:
next_state = None
experience_replay.push(state, action, next_state, r)
#total_experience.add(total_experience.get_max(), (w_state, action, r, next_w_state, done))
state = next_state
if global_step > self.cfg.TRAIN_START and len(
experience_replay) > self.cfg.BATCH_SIZE:
transitions = experience_replay.sample()
batch = Transition(*zip(*transitions))
non_final_mask = torch.tensor(tuple(
map(lambda s: s is not None, batch.next_state)),
device=self.device,
dtype=torch.uint8)
non_final_next_states = torch.cat(
[s for s in batch.next_state if s is not None])
state_batch = torch.cat(batch.state).to(self.device)
action_batch = torch.cat(batch.action).to(self.device)
reward_batch = torch.cat(batch.reward).to(self.device)
# reward standardization
if self.cfg.STD_REWARDS:
reward_batch = (reward_batch - reward_batch.mean()) / (
reward_batch.std() +
np.finfo(np.float32).eps.item())
state_action_values = self.model(state_batch).gather(
1, action_batch)
next_state_values = torch.zeros(self.cfg.BATCH_SIZE,
device=self.device)
if self.cfg.DOUBLE_DQN:
_, next_state_actions = self.model(
non_final_next_states.to(self.device)).max(
1, keepdim=True)
next_state_values[non_final_mask] = self.target_model(
non_final_next_states.to(self.device)).gather(
1, next_state_actions).squeeze(1).detach()
else:
next_state_values[non_final_mask] = self.target_model(
non_final_next_states.to(
self.device)).max(1)[0].detach()
expected_state_action_values = (
next_state_values * self.cfg.GAMMA) + reward_batch
if self.cfg.HUBER_LOSS:
loss = F.smooth_l1_loss(
state_action_values,
expected_state_action_values.unsqueeze(1))
else:
loss = F.mse_loss(
state_action_values,
expected_state_action_values.unsqueeze(1))
# training op
self.optimizer.zero_grad()
loss.backward()
for param in self.model.parameters():
param.grad.data.clamp_(-1, 1)
self.optimizer.step()
self.logger.log_value('loss',
global_step,
loss.item(),
print_value=False,
to_file=False)
#for i in range(self.cfg.BATCH_SIZE):
# total_experience.update(b_i[i], errors[i])
# update target weights
if (global_step % self.cfg.TARGET_UPDATE) == 0:
self.target_model.load_state_dict(self.model.state_dict())
self.logger.log_episode('DQN agent', e, self.env.total_reward)
if (e % self.cfg.SAVE_STEP) == 0:
torch.save(self.target_model.state_dict(), self.ckpt_path)
self.logger.log_variables()
torch.save(self.target_model.state_dict(), self.ckpt_path)
self.logger.log_variables()
def make_folders(self):
"""
Creates folders for the experiment logs and ckpt
"""
os.makedirs(os.path.join(self.experiment_path, 'ckpt'))
os.makedirs(os.path.join(self.experiment_path, 'logs'))
""" instantiate PiPAD + RoPi sensors """ pipad_led = PiPadLED() pipad_buttons = PiPadButtons() pipad_stick = PiPadStick() pipad_rotary = PiPadRotary() ropi_led = RoPiLED(_ropi['remote_address']) ropi_buzzer = RoPiBuzzer(_ropi['remote_address']) ropi_servo = RoPiServo(_ropi['remote_address']) ropi_track = RoPiTrack(_ropi['remote_address']) ropi_distance = RoPiDistance(_ropi['remote_address']) #ropi_screen = RoPiScreen(_ropi['remote_address']) ropi_motor = RoPiMotor(_ropi['remote_address']) # LOGGING """ instantiate logger """ log = Logger(_pipad['DEVEL_LOG']) # GLOBALS """ global variables """ rotary_counter_lr = 0 rotary_current_counter_lr = 0 rotary_counter_ud = 0 rotary_current_counter_ud = 0 rotary_reverse = 0 stick_autonomous = False stick_position = 0 # COLOR A LED """ color a specified RGBLEB *to_color* *for_time* """
post_list.append(tweet)
logger.info("{0} - {1} : {2}".format(
id, screen_name, text))
ignore_list.append(id)
logger.info("Got {0} results".format(c))
except tweepy.TweepError as e:
logger.exception(
"Could not connect to Twitter - are your credentials correct?")
if __name__ == '__main__':
Config.load('config.json')
logger = Logger('twitter-contestant')
# Intilialize Tweepy
auth = tweepy.OAuthHandler(Config.consumer_key, Config.consumer_secret)
auth.set_access_token(Config.access_token_key, Config.access_token_secret)
api = tweepy.API(auth,
wait_on_rate_limit=True,
wait_on_rate_limit_notify=True)
# Initialize ignorelist
ignore_list = IgnoreList("ignorelist")
# Initialize scheduler
scheduler = BlockingScheduler()
# First run
class Driver:
def __init__(self):
self.state = State(0)
self.params = Params()
self.status = Status(self.params)
self.sleep_time = 1
self.pwm_read = PwmRead(self.params.pin_mode_in,
self.params.pin_servo_in,
self.params.pin_thruster_in)
self.pwm_out = PwmOut(self.params.pin_servo_out,
self.params.pin_thruster_out)
self.pid = PositionalPID()
self.logger = Logger()
self.logger.open()
def load(self, filename):
print('loading', filename)
f = open(filename, "r")
line = f.readline()
line = f.readline()
self.state.time_limit = int(line.split()[1]) # Time Limit
line = f.readline()
self.sleep_time = float(line.split()[1]) # Sleep time
line = f.readline()
line = f.readline()
line = f.readline()
p = float(line.split()[1]) # P
line = f.readline()
i = float(line.split()[1]) # I
line = f.readline()
d = float(line.split()[1]) # D
self.pid.setPID(p, i, d)
line = f.readline()
line = f.readline()
line = f.readline()
num = int(line.split()[1]) # Number of waypoints
line = f.readline()
for i in range(num):
line = f.readline()
self.status.waypoint.addPoint(float(line.split()[0]),
float(line.split()[1]))
f.close()
return
def doOperation(self):
while self.state.inTimeLimit():
self.readPWM()
self.readGps()
mode = self.getMode()
if mode == 'RC':
self.remoteControl()
elif mode == 'AN':
self.autoNavigation()
self.outPWM()
self.printLog()
time.sleep(self.sleep_time)
return
def getMode(self):
return self.status.mode
def updateMode(self):
mode_duty_ratio = self.pwm_read.pulse_width[0]
if mode_duty_ratio < 1500:
self.status.mode = 'RC'
elif mode_duty_ratio >= 1500:
self.status.mode = 'AN'
return
def readGps(self):
self.status.readGps()
self.updateMode()
#if self.status.isGpsError():
#self.status.mode = 'RC'
return
def updateStatus(self):
status = self.status
status.calcTargetDirection()
status.calcTargetDistance()
status.updateTarget()
return
def readPWM(self):
self.pwm_read.measurePulseWidth()
self.pwm_out.servo_pulsewidth = self.pwm_read.pulse_width[1]
self.pwm_out.thruster_pulsewidth = self.pwm_read.pulse_width[2]
return
def outPWM(self):
self.pwm_out.updatePulsewidth()
return
def autoNavigation(self):
self.updateStatus()
boat_direction = self.status.boat_direction
target_direction = self.status.target_direction
servo_pulsewidth = self.pid.getStepSignal(target_direction,
boat_direction)
self.pwm_out.servo_pulsewidth = servo_pulsewidth
self.pwm_out.thruster_pulsewidth = 1880
return
def remoteControl(self):
# Do nothing
return
def printLog(self):
timestamp_string = self.status.timestamp_string
mode = self.getMode()
latitude = self.status.latitude
longitude = self.status.longitude
speed = self.status.speed
direction = self.status.boat_direction
servo_pw = self.pwm_out.servo_pulsewidth
thruster_pw = self.pwm_out.thruster_pulsewidth
t_direction = self.status.target_direction
t_distance = self.status.target_distance
target = self.status.waypoint.getPoint()
t_latitude = target[0]
t_longitude = target[1]
print(timestamp_string)
print(
'[%s MODE] LAT=%.7f, LON=%.7f, SPEED=%.2f [km/h], DIRECTION=%lf' %
(mode, latitude, longitude, speed, direction))
print('DUTY (SERVO, THRUSTER): (%lf, %lf) [us]' %
(servo_pw, thruster_pw))
print('TARGET (LATITUDE, LONGITUDE): (%lf, %lf)' %
(t_latitude, t_longitude))
print('TARGET (DIRECTION, DISTANCE): (%lf, %lf [m])' %
(t_direction, t_distance))
print('')
log_list = [
timestamp_string,
mode,
latitude,
longitude,
direction,
]
self.logger.write(log_list)
return
def finalize(self):
self.logger.close()
self.pwm_out.finalize()
return
def index():
logger = Logger.getInstance()
user_ip = request.remote_addr
logger.log("index", user_ip)
return render_template("index.html")
def main():
usage = "Usage: %prog [options] <package name>"
parser = ArgumentParser(usage)
parser.add_argument("-s",
"--spec-path",
dest="specPath",
default="../../SPECS")
parser.add_argument("-l",
"--log-path",
dest="logPath",
default="../../stage/LOGS")
parser.add_argument("-a",
"--source-rpm-path",
dest="sourceRpmPath",
default="../../stage/SRPMS")
parser.add_argument("-j",
"--output-dir",
dest="outputDirPath",
default="../../stage/")
parser.add_argument("-c",
"--pullsources-config",
dest="pullsourcesConfig",
default="pullsources.conf")
parser.add_argument("-f",
"--pkg-blacklist-file",
dest="pkgBlacklistFile",
default=None)
parser.add_argument("-p",
"--generate-pkg-list",
dest="generatePkgList",
default=False,
action="store_true")
parser.add_argument("-y",
"--generate-yaml-files",
dest="generateYamlFiles",
default=False,
action="store_true")
options = parser.parse_args()
errorFlag = False
cmdUtils = CommandUtils()
try:
if not os.path.isdir(options.logPath):
cmdUtils.runCommandInShell("mkdir -p " + options.logPath)
logger = Logger.getLogger(options.logPath + "/generateYamlFiles")
if options.generateYamlFiles:
if (options.pkgBlacklistFile is not None
and options.pkgBlacklistFile != ""
and not os.path.isfile(options.pkgBlacklistFile)):
logger.error("Given package blacklist file is not valid:" +
options.pkgBlacklistFile)
errorFlag = True
if not os.path.isdir(options.specPath):
logger.error("Given Specs Path is not a directory:" +
options.specPath)
errorFlag = True
if not os.path.isdir(options.sourceRpmPath):
logger.error("Given SRPM Path is not a directory:" +
options.sourceRpmPath)
errorFlag = True
if options.generateYamlFiles and not os.path.isfile(
options.pullsourcesConfig):
logger.error("Given Source config file is not a valid file:" +
options.pullsourcesConfig)
errorFlag = True
if errorFlag:
logger.error(
"Found some errors. Please fix input options and re-run it.")
sys.exit(1)
if options.generateYamlFiles:
if not os.path.isdir(options.outputDirPath):
cmdUtils.runCommandInShell("mkdir -p " + options.outputDirPath)
constants.setSpecPath(options.specPath)
constants.setSourceRpmPath(options.sourceRpmPath)
constants.setLogPath(options.logPath)
constants.setPullSourcesConfig(options.pullsourcesConfig)
constants.initialize()
# parse SPECS folder
SPECS()
if options.generatePkgList:
buildPackagesList(options.outputDirPath + "/packages_list.csv")
elif options.generateYamlFiles:
blackListPkgs = readBlackListPackages(options.pkgBlacklistFile)
buildSourcesList(options.outputDirPath, blackListPkgs, logger)
buildSRPMList(options.sourceRpmPath, options.outputDirPath,
blackListPkgs, logger)
except Exception as e:
print("Caught Exception: " + str(e))
traceback.print_exc()
sys.exit(1)
sys.exit(0)
class DrebinUtil:
datasetPath = "" #PATH DEL DATASET
testPath = "" #PATH DOVE INSERIRE GLI APK OFFUSCATI
dictionary = "" #PATH DEL DICTIONARY
logLevel = 3
__logger = ""
##The constructor:
# @param datasetPath = path of the dataset;
# @param testPath = path of the workspace;
# @param dictionaryPath = path of the drebin dictionary;
def __init__(self, datasetPath, testPath, dictionaryPath):
self.datasetPath = datasetPath
self.testPath = testPath
self.dictionary = dictionaryPath
self.logger = Logger(self.logLevel)
##This method read from the dictionary the family of a sample;
# @param fileName = the name of the sample to find the family ;
def readMalwareFamily(self, fileName):
bashArray = ["grep", fileName, self.dictionary]
process = subprocess.Popen(bashArray,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
output, error = process.communicate()
output = str(output)
if (error or not output):
malwareFamily = "UKNOWN"
else:
malwareFamilyList = output.split(",")
malwareFamily = malwareFamilyList[1]
malwareFamily = malwareFamily.replace("\r", " ")
malwareFamily = malwareFamily.replace("\n", " ")
self.logger.log("INFO", "MALWARE FAMILY: " + malwareFamily)
return (malwareFamily)
##This method change the name of a file (used to add .apk extension);
# @param fileName = the original filename;
# @param newName = the new name;
def prepareFile(self, fileName, newName):
bashCommand = "mv " + self.datasetPath + fileName + " " + self.testPath + newName
process = subprocess.Popen(bashCommand.split(),
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
output, error = process.communicate()
if (error):
self.logger.log("ERROR", "ERROR PREPARING APK FILE")
return (-1)
else:
self.logger.log("INFO", "ADD APK EXTENSION: " + newName)
return (0)
def cleanBadApk(self, newName, mapName, obfuscatedName, packageName):
bashCommand = "rm " + self.testPath + newName
try:
subprocess.check_output(bashCommand,
stderr=subprocess.STDOUT,
shell=True)
self.logger.log("INFO", "REAL APK REMOVED")
except subprocess.CalledProcessError:
self.logger.log("ERROR", "ERROR REMOVING REAL APK")
bashCommand = "rm " + self.testPath + mapName
try:
subprocess.check_output(bashCommand,
stderr=subprocess.STDOUT,
shell=True)
self.logger.log("INFO", "MAP FILE REMOVED")
except subprocess.CalledProcessError:
self.logger.log("ERROR", "ERROR REMOVING MAP FILE")
bashCommand = "rm " + self.testPath + obfuscatedName
try:
subprocess.check_output(bashCommand,
stderr=subprocess.STDOUT,
shell=True)
self.logger.log("INFO", "OBFUSCATED APK REMOVED")
except subprocess.CalledProcessError:
self.logger.log("ERROR", "ERROR REMOVING OBFUSCATED APK")
def moveGoodApk(self, newName, mapName, obfuscatedName, packageName,
goodAPKPath):
self.logger.log("INFO", "THIS APK IS GOOD, MOVING TO NEW LOCATION")
folder = goodAPKPath + "/" + packageName
bashCommand = "mkdir " + folder
try:
subprocess.check_output(bashCommand,
stderr=subprocess.STDOUT,
shell=True)
self.logger.log("INFO", "CREATED DIRECTORY: " + folder)
except subprocess.CalledProcessError:
self.logger.log("ERROR", "ERROR CREATING DIRECTORY: " + folder)
bashCommand = "mv " + self.testPath + newName + " " + folder + "/" + newName
try:
subprocess.check_output(bashCommand,
stderr=subprocess.STDOUT,
shell=True)
self.logger.log("INFO", "REAL APK MOVED")
except subprocess.CalledProcessError:
self.logger.log("ERROR", "ERROR MOVING REAL APK")
bashCommand = "mv " + self.testPath + mapName + " " + folder + "/" + mapName
try:
subprocess.check_output(bashCommand,
stderr=subprocess.STDOUT,
shell=True)
self.logger.log("INFO", "MAP FILE MOVED")
except subprocess.CalledProcessError:
self.logger.log("ERROR", "ERROR MOVING MAP FILE")
bashCommand = "mv " + self.testPath + obfuscatedName + " " + folder + "/" + obfuscatedName
try:
subprocess.check_output(bashCommand,
stderr=subprocess.STDOUT,
shell=True)
self.logger.log("INFO", "OBFUSCATED APK MOVED")
except subprocess.CalledProcessError:
self.logger.log("ERROR", "ERROR MOVING OBFUSCATED APK")
class InfraredSensor:
DR = 16
DL = 19
interrupted = False
thread = None
on = False
left = 1
right = 1
def __init__(self, client, service_name, debug=False):
self.client = client
self.serviceName = service_name
self.logger = Logger("InfraredSensor", debug)
self.source = Observable.interval(50).map(lambda i: self.get_state())
GPIO.setup(self.DR, GPIO.IN, GPIO.PUD_UP)
GPIO.setup(self.DL, GPIO.IN, GPIO.PUD_UP)
self.client.publish(self.serviceName + "/state/ir-sensor/state", "OFF",
1, True)
self.client.publish(self.serviceName + "/state/ir-sensor/state/left",
"OFF", 1, True)
self.client.publish(self.serviceName + "/state/ir-sensor/state/right",
"OFF", 1, True)
self.client.message_callback_add(
self.serviceName + "/control/ir-sensor/#", self.on_message)
def on_message(self, client, userdata, msg):
self.logger.info(msg.topic + ": " + msg.payload)
try:
path = msg.topic.split("/")
if len(path) > 1 and path[0] == self.serviceName and path[
1] == "control": # mutinus/control/#
if len(path) > 2 and path[
2] == "ir-sensor": # mutinus/control/ir-sensor
self.logger.info(msg.payload)
if msg.payload == "ON":
self.start()
else:
self.stop()
except:
self.logger.error("Unexpected Error!")
traceback.print_exc()
def get_state(self):
left = GPIO.input(self.DL)
right = GPIO.input(self.DR)
return [left, right]
def subscribe(self, on_next):
return self.source.subscribe(
on_next=on_next,
on_error=lambda e: self.logger.error(str(e)),
on_completed=lambda: self.logger.info("Subscription completed"))
def looper(self):
while not self.interrupted:
try:
state = self.get_state()
self.logger.debug("Left: " + str(state[0]) + ", Right: " +
str(state[1]))
if state[0] != self.left:
self.left = state[0]
payload = "ON" if self.left == 0 else "OFF"
self.client.publish(
self.serviceName + "/state/ir-sensor/left", payload, 0,
False)
if state[1] != self.right:
self.right = state[1]
payload = "ON" if self.right == 0 else "OFF"
self.client.publish(
self.serviceName + "/state/ir-sensor/right", payload,
0, False)
time.sleep(0.05)
except:
self.logger.error("Unexpected Error!")
traceback.print_exc()
self.logger.info("Exiting looper")
def start(self):
if self.thread is not None:
self.client.publish(self.serviceName + "/state/ir-sensor", "ON", 1,
True)
self.interrupted = False
self.thread = threading.Thread(target=self.looper)
self.thread.start()
return self.thread
def stop(self):
self.interrupted = True
if self.thread is not None:
self.client.publish(self.serviceName + "/state/ir-sensor", "OFF",
1, True)
self.thread.join(5)
self.thread = None
# -*- coding:utf-8 -*-
# @Time： 2020/7/11 10:31 PM
# @Author: dyf-2316
# @FileName: getData.py
# @Software: PyCharm
# @Project: Comment_Sentiment_Analysis
# @Description: get data from html/json
import re
from Logger import Logger
from config import *
from WebCrawler.getPageSource import get_page_html, get_json_data
mylogger = Logger("gatData").logger
def get_search_url(keywords):
"""
通过搜索界面URL与关键字的拼接，获取一个关键字商品搜索页面的URL。
:param keywords: (str) 想要查询商品的关键字
:return: (str) 含有关键字的商品搜索界面的URL
"""
search_url = SEARCH_URL.format(keywords)
# logger传入message必须是字符串（用format或%拼接好），与print相区别
mylogger.debug("获取搜索界面URL -- {}".format(search_url))
return search_url
product_already_get = ['100002076057', '47775809660', '51961736081', '51961736083', '51961736082',
def A1(F,G): P = IMPLICATION(F,IMPLICATION(G,F)) Logger(P,'A1 for '+F+' and '+G) return P
