def word_repost_relationship(temp_dir, searchList, breakpos=None): # 据进程名生成日志 name = 'getRepost_' + str(os.getpid()) logger = getLogger(name) # 每个进程维护一个sublist转发关系的层级目录(临时) level_dir = temp_dir + 'temp/' os.mkdir(level_dir) # 断点处理 if not breakpos: # 生成写文件 repost_file = temp_dir + name + '.csv' repost_writer = csvWriter(repost_file, repost=True) else: repost_file = temp_dir + breakpos['repost_file'] repost_writer = csvWriter(repost_file, repost=True, breakpos=True) # 先爬取完断点id,再对余下id按常规爬取 get_repost_relationship(breakpos['center_bw_id'], repost_writer, level_dir, logger, breakpos) searchList = searchList[1:] # 常规爬取 logger.info('Strat getting repost...') for id in searchList: get_repost_relationship(id, repost_writer, level_dir, logger) logger.info('Finish!') # 删除临时目录 shutil.rmtree(level_dir)
def app(event_loop): # main -> put arguments. logger = getLogger('main') parser = argument_parser() try: prepare_service(logger, event_loop, parser) except KeyboardInterrupt: logger.info('service end, reason: keyboard interrupt')
def word_spider(searchlist): # 加载设置文件,获取数据输出路径和检索词 config = load_config() hot_dir = config['hot_dir'] topic_dir = config['topic_dir'] repost_dir = config['repost_dir'] # 根据规定日志目录创建目录实例 name = multiprocessing.current_process().name logger = getLogger(name) topic_dir += name + '_' # 记录载入检索词列表的次数 epoch = 1 while True: # 对每一个词爬取相关微博和各微博的转发关系 for wd in searchlist: logger.info(f'EPOCH: {epoch}. Keyword: {wd}. Start crawling ...') search_file = hot_dir + 'search_result_' + str(wd) + '.csv' repost_file = repost_dir + 'repost_Relationship_' + str(wd) + '.csv' # 创建两个写的对象,同时创建文件 search_writer = csvWriter(search_file, search=True) repost_writer = csvWriter(repost_file, repost=True) # 获取该检索词的所有相关微博,至多能获取1000条 get_query_info(wd, search_writer, logger) # 获取相关微博id组成的列表 idList = search_writer.get_idList() # 获取各相关微博的转发关系 for bw_id in idList: get_repost_relationship(bw_id, repost_writer, logger) repost_writer.drop_duplicates() # 获取该词相关所有话题作为之后的检索词 get_more_topic(wd, epoch, topic_dir, logger) # 结束一轮检索爬取 # 获取新检索词列表 filename = topic_dir + 'Topics_' + str(epoch) + '.csv' with open(filename, 'r', encoding='utf-8-sig') as f: rows = csv.reader(f) searchlist = [row[0].strip() for row in rows] # 删除中间文件 os.remove(filename) epoch += 1
def read(self, file_dir, file_name): log = logger.getLogger() file_path = file_dir + file_name try: if os.path.exists(file_path): with open(file_path, 'r') as f: meta = json.loads(f.read()) self.__dict__ = meta log.debug('loading %s' % file_path) return True else: log.debug('No {} file founded. Creating new one.'.format(file_name)) JSONUtils.write(self, file_dir, file_name) return False except Exception as e: traceback.print_exc() print("read an JSON file to path error: %s, Case: %s" % (file_path, e)) return False
import os import copy import time import pickle import xml.etree.ElementTree as ET from os.path import expanduser import rospy import tf from config import ROS_Launch_File, Template_Turtlebot_Launch, Template_Rosbot_Launch, Map_Dir, Launch_Max_Try, Nav_Process_Pool, Robot_Model, Pos_Value_Splitter from utils.ros_utils import checkRobotNode, shell_open, initROSNode #from utils.logger import logger from utils.logger import getLogger logger = getLogger('Turtlebot_Launcher') logger.propagate = False class Turtlebot_Launcher(): def __init__(self, inspection_id, siteid, robots): self.robots = robots self.siteid = siteid self.inspection_id = inspection_id self.trans_listener = None def launch(self): launched = False for i in range(Launch_Max_Try): logger.info(
def logger(self): if self._logger is None: self._logger = getLogger('node') return self._logger
# -*- coding: utf-8 -*- """ @author: xingxingzaixian @create: 2020/9/6 @description: """ from django.contrib.auth import backends from django.db.models import Q from django.contrib.auth.backends import UserModel from rest_framework.exceptions import AuthenticationFailed from utils.auth.jwt_util import JwtUtil from utils.logger import getLogger logger = getLogger('auth') class UserBackend(backends.ModelBackend): def authenticate(self, request, username=None, password=None, **kwargs): try: user = UserModel.objects.get(Q(username=username) | Q(telephone=username)) if user.check_password(password): token = JwtUtil.gen_jwt_token(user) user.token = token return user except UserModel.DoesNotExist as e: logger.error(f'[username={username}]: {e}') raise AuthenticationFailed(detail="User authentication failure")
import requests from discord import Embed, Message from discord.ext import commands from requests import Session from bot.cog import BaseCog from utils.config import Config, ConfigData from utils.googledrive import GoogleDrive as Drive from utils.googledrive import GoogleDriveFile from utils.logger import getLogger drive: Drive = Drive() config: ConfigData = Config.read() r: Session = requests.Session() logger: Logger = getLogger(__name__) class CmdFanart(BaseCog): @commands.group() async def fanart(self, ctx): if ctx.invoked_subcommand is None: await ctx.send('subcommand is required') @fanart.command() async def list(self, ctx): logger.info("fetch VTuberFanartCrawler image count") embed = Embed(title="Fanart Image Count", description="fetch image. please wait for while...", color=0x80fffd)
import copy import json import time import redis from kafka import KafkaProducer import config from utils.logger import getLogger logger = getLogger('utils-kafka') logger.propagate = False status_producer = KafkaProducer( bootstrap_servers=config.Kafka_Brokers, compression_type='gzip', value_serializer=lambda x: json.dumps(x).encode()) redis_connector = redis.Redis(host=config.redis_host, port=config.redis_port, db=0) # just an example, you don't have to follow this format task_status_payload = { "inspection_id": 3, "task_type": 0, "site_id": "site01", 'timestamp': 1599033481, "robot": { "robot_id": "robot03", "checkpoint_no": 2,
import copy import datetime from influxdb import InfluxDBClient import config from utils.logger import getLogger logger = getLogger('utils-tsdb') body_pos = { 'measurement': config.Table_Name_Robot_Pos, 'time': 0, 'tags': { 'robot_id': 0, 'inspection_id': 0, 'site_id': 0, }, 'fields': { 'pos_x': 0, 'pos_y': 0, 'pos_angle': 0 } } body_event = { 'measurement': config.Table_Name_Robot_Event, 'time': 0, 'tags': { 'robot_id': 0, 'inspection_id': 0, 'site_id': 0,
import redis from kafka import KafkaProducer import json from utils.logger import getLogger logger = getLogger('test_redis_kafka') logger.propagate = False robot_id = 'tb3_0' task_type = 30 site_id = 'bj02' inspection_id = 562 def test_redis_service(): #redis redis_host = "123.1127.237.146" redis_port = "6379" robot_position_payload = { "timestamp": 1599033481, "robot_id": 'hello_a', "inspection_id": 0, "site_id": 0, "location": '0-0-0' } redis_connector = redis.Redis(host=redis_host, port=redis_port, db=0) try: redis_connector.hmset(0, robot_position_payload) logger.info('Redis operation : send robot pos {}#{}'.format(10, 2))
import torch import torch.optim as optim import torch.multiprocessing as mp import torch.nn as nn import torch.nn.functional as F import tensorboard_logger as tb import my_optim from envs import create_atari_env from model import ActorCritic from train import train from test import test from utils import logger from utils.shared_memory import SharedCounter logger = logger.getLogger('main') # Based on # https://github.com/pytorch/examples/tree/master/mnist_hogwild # Training settings parser = argparse.ArgumentParser(description='A3C') parser.add_argument('--lr', type=float, default=0.0001, metavar='LR', help='learning rate (default: 0.0001)') parser.add_argument('--gamma', type=float, default=0.99, metavar='G', help='discount factor for rewards (default: 0.99)')
# export PYTHONPATH="/home/whx/workspace/work/python_code/autotext:{$PYTHONPATH}" import tensorflow as tf import numpy as np from datamanager import dataload from rnn.model import RNN from config import SEQ_LEN import argparse import os from utils import dateUtil from utils import fileUtil from utils import logger from utils import configParse from rnn import predict _LOG = logger.getLogger('rnn_train') def saveModel(model, saveModelDir, signatures=None): if saveModelDir is None: return tagDir = os.path.join(saveModelDir, dateUtil.getNow(format='%Y%m%d%H')) tagDir = fileUtil.getDir(tagDir, op=2) if type(signatures) == dict: tf.saved_model.save(model, tagDir, signatures) else: tf.saved_model.save(model, tagDir) def loadModel(loadModelDir, flag): tagDir = os.path.join(loadModelDir, flag)
def word_get_query_info(wd, writer): logger = getLogger('getQuery') logger.info(f'Keyword: {wd}. Start crawling ...') get_query_info(wd, writer, logger)
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ''' # TurtleBot must have minimal.launch & amcl_demo.launch # running prior to starting this script # For simulation: launch gazebo world & amcl_demo prior to run this script import rospy from move_base_msgs.msg import MoveBaseAction, MoveBaseGoal import actionlib from actionlib_msgs.msg import * from geometry_msgs.msg import Pose, Point, Quaternion from config import Wait_For_GoToPose_Time, DEBUG from utils.logger import getLogger logger = getLogger('GoToPose') class GoToPose(): def __init__(self, inspection_id, robot_id): self.inspection_id = inspection_id self.robot_id = robot_id self.msg_head = 'inspection:{} robot: {}: '.format( inspection_id, robot_id) self.goal_sent = False # What to do if shut down (e.g. Ctrl-C or failure) rospy.on_shutdown(self.shutdown)
import re import time import random import requests from lxml import etree from datetime import datetime from config.base_config import user_agent from utils.util import filter_data from utils.logger import getLogger from data_structure.singleton_structure_content import new_content logger = getLogger('zhuyuan') headers = {'User-Agent': random.choice(user_agent)} def xueyansuo(blh): url = 'http://192.168.2.8:8055/Report/SearchReport' json_data = { "bah": blh, "startTime": "2007-01-01", "endTime": datetime.now().strftime('%Y-%m-%d'), "Types": ["XYS", "PS", "SZB", "LIS-RST"], } resp = requests.post(url, json=json_data) if len(resp.text) > 10: doc = etree.HTML(resp.text) result = doc.xpath('//tr') xys = []
import time import math import rospy from geometry_msgs.msg import Twist from nav_msgs.msg import Odometry from tf.transformations import euler_from_quaternion from utils.logger import getLogger logger = getLogger('RotateController') logger.propagate = False PI = 3.1415926535897 class RotateController(): def __init__(self, inspection_id, robot_id): self.inspection_id = inspection_id self.robot_id = robot_id self.msg_head = 'inspection:{} robot: {}: '.format(inspection_id,robot_id) # self.sub = rospy.Subscriber ('/{}/odom'.format(self.robot_id), Odometry, self.get_rotation) self.rotate_pub = rospy.Publisher('/{}/cmd_vel'.format(self.robot_id), Twist, queue_size=1) self.rotate_command =Twist() self.roll = 0.0 self.pitch = 0.0 self.yaw = 0.0 self.ctrl_c = False self.rate = rospy.Rate(10)
def logger(self): if self._logger is None: self._logger = getLogger('storage') return self._logger
import threading import time import json from kafka import KafkaConsumer from config import Task_Type, Kafka_Brokers, Task_Topic from utils.ros_utils import killNavProcess, initROSNode from tasks.inspection import execInspection from utils.logger import getLogger logger = getLogger('main') logger.propagate = False task_subscriber = KafkaConsumer( bootstrap_servers=Kafka_Brokers, group_id="robot_controller2", auto_offset_reset="earliest") # def execTaskLoop(): # while True: # task = getTasksFromMsgQueue() # if task is None: # time.sleep(1) # continue # task_type, task_data = task[0], task[1] # if task_type == Task_Type["Task_Inspection"]: # inspection_id = int(task_data['inspection_id']) # task_name = 'inpsection: {}'.format(inspection_id) # logger.info('start inspection task: {}'.format(task_name)) # task = threading.Thread(name=task_name, target=execInspection, args=(task_data,))
from utils.logger import getLogger logger = getLogger('InspectionMonitor') logger.propagate = False from config import Inspection_Status_Codes Worker_Status = {'working': 0, 'idle': 1, 'failed': 3} class InspectionMonitor(object): """ InspectionMonitor only records the status of on-going inspection task, which also records the workers (robot)'s three status: 'working', 'idel' and 'failed', but dose not record the robot's entrance and leave events """ inspection_monitor = None def __init__(self): self.task_list = {} @staticmethod def getInspectionMonitor(): if InspectionMonitor.inspection_monitor is None: InspectionMonitor.inspection_monitor = InspectionMonitor() return InspectionMonitor.inspection_monitor def addTask(self, inspection_id, site_id, robot_ids): self.task_list[inspection_id] = { 'site_id': site_id,
from os.path import dirname import tornado.web import tornado.ioloop from models import conn from utils.config import config from utils.logger import getLogger from views.IndexHandler import IndexHandler from views.LoginHandler import LogoutHandler from views.LoginHandler import QQLoginHandler from views.LoginHandler import DirectLoginHandler logger = getLogger("chat2all.engine") def make_app(): """ make an app instance to start the server """ settings = {'static_path': join(dirname(__file__), 'static'), 'template_path': join(dirname(__file__), 'templates')} cookie_secret = config.get('cookie_secret') app = tornado.web.Application([(r"/qq_redirect/", QQLoginHandler), (r"/login/", DirectLoginHandler), (r"/logout/", LogoutHandler), (r"/", IndexHandler)], cookie_secret=cookie_secret, **settings)
def logger(self): if self._logger is None: self._logger = getLogger('statedb') return self._logger
import math import os import sys import torch import torch.nn.functional as F import torch.optim as optim from torch.autograd import Variable from torchvision import datasets, transforms from utils import logger logger = logger.getLogger(__name__) def ensure_shared_grads(model, shared_model): for param, shared_param in zip(model.parameters(), shared_model.parameters()): if shared_param.grad is not None: return shared_param._grad = param.grad def train(rank, args, shared_model, Model, make_env, gl_step_count, optimizer=None): torch.manual_seed(args.seed + rank)
from django.utils.deprecation import MiddlewareMixin from utils.logger import getLogger logger = getLogger('middle') class ExceptionMiddleware(MiddlewareMixin): def process_exception(self, request, exception): """ 记录所有的异常日志 """ logger.error(f'URL=[{request.path}], {exception}')
# 在requests headers中,禁用删除If - Modified-Since 和If-None-Natch 这两项 import time import requests from jsonpath import jsonpath from utils.decrypt_data import decrypt_data from utils.logger import getLogger from utils.crack_password import crack_pwd from utils.util import filter_data, quchong logger = getLogger('new_his') USER = '******' PASSWORD = '******' ACCESS_TOKEN = None K1 = None K2 = None def login(): global ACCESS_TOKEN, K1, K2 url = 'http://his.zheyi.com/app-sso/oauth/token' params = { 'grant_type': 'password', 'username': USER, 'password': crack_pwd(PASSWORD), 'verifyCode': '', 'sessionId': '', } basic_authorize = { 'Authorization': 'Basic bm9idWc6Z2l2ZW1lZml2ZQ==',
import sys import time import torch import torch.nn.functional as F import torch.optim as optim import tensorboard_logger as tb from envs import create_atari_env from model import ActorCritic from torch.autograd import Variable from torchvision import datasets, transforms from collections import deque from utils import logger logger = logger.getLogger('test') def test(rank, args, shared_model, gl_step_cnt): torch.manual_seed(args.seed + rank) env = create_atari_env(args.env_name) env.seed(args.seed + rank) model = ActorCritic(env.observation_space.shape[0], env.action_space) model.eval() state = env.reset() state = torch.from_numpy(state) reward_sum = 0
# -*- coding:utf-8-*- from threading import Thread from utils.logger import getLogger logger = getLogger('utils') def quchong(array): seen = set() new_array = [] for a in array: temp = tuple(a.items()) if temp not in seen: seen.add(temp) new_array.append(a) return new_array def capital_to_lower(doc): assert isinstance(doc, dict) new = {} for key, value in doc.items(): new[key.lower()] = value return new def filter_data(ybzl, obj_data): # assert len(obj_data) == 5 if isinstance(ybzl, dict) and ybzl.get(20104):
(all_labels, labels.detach().cpu().numpy()), axis=0) all_logits = np.concatenate( (all_logits, logits.detach().cpu().numpy()), axis=0) acc = accuracy(all_logits, all_labels) f1 = f1_score(all_logits, all_labels) return np.array(loss).mean(), acc, f1 if __name__ == "__main__": # 创建存储目录 if not os.path.exists(args.save_dir): os.makedirs(args.save_dir) logger = getLogger(__name__, os.path.join(args.save_dir, 'log.txt')) if 'roberta' in args.model_type: MatchModel = RobertaMatchModel Tokenizer = RobertaTokenizer elif 'albert' in args.model_type: MatchModel = AlbertMatchModel Tokenizer = AlbertTokenizer elif 'bert' in args.model_type: MatchModel = BertMatchModel Tokenizer = BertTokenizer # elif 'ernie' in args.model_type: # MatchModel = # Tokenizer = AutoTokenizer if args.do_train:
import sys import json import rospy import tf from kafka import KafkaProducer import config from utils.ros_utils import initROSNode from utils.logger import getLogger logger = getLogger('taskcenter_simulator') logger.propagate = False task_producer = KafkaProducer( bootstrap_servers=config.Kafka_Brokers, compression_type='gzip', value_serializer=lambda x: json.dumps(x).encode()) def startTask(): # task_body = { # "task_type": 0, # 0 for Task_Inspection # "inspection_id": 6, # "site_id": "bj01", # "robots": [ # { # "robot_id": "rosbot1", #"tb3_0", # "model": "rosbot2_pro", #waffle_pi", # "original_pos": "0.0#0.0#0.0", # x-y-angle # "subtasks": [
from data_structure.singleton_structure_content import new_content from huayan import HuayanData from lis_new import LisData from menzhen import MenZhenData from new_his import NewHis from yihui import YihuiSystem from yingxiang import YingXiang from zhuyuan import HisSystem from utils.logger import getLogger import warnings import pymongo warnings.filterwarnings("ignore") import pymssql logger = getLogger('xuetou') # 数据存储 client = pymongo.MongoClient(MONGO_HOST) db = client[MONGO_DB] collection = db[COLLECTION] # collection = db['origin'] connect = pymssql.connect(host=host, port=port, user=user, password=password, database=database) cursor = connect.cursor() UPDATE_SUCCESS_IDS = [] with open('config/jianyan_field.json', 'r') as f: jianyan_config = json.load(f)
import threading import time from config import Inspection_Status_Codes, Enable_Influx, Pos_Value_Splitter, Robot_Model from navigation.turtlebot_launch import Turtlebot_Launcher from navigation.turltlebot_cruise import runRoute from navigation.turtlebot_robot_status import setRobotWorking, setRobotIdel, isRobotWorking, isInspectionRunning, isInspectionRepeated from utils.msg_utils import sendTaskStatusMsg, sendDiscoveryStopRecords from utils.ros_utils import killNavProcess, checkMapFile from monitor import InspectionMonitor from utils.logger import getLogger logger = getLogger('execInspection') logger.propagate = False inspection_monitor = InspectionMonitor.getInspectionMonitor() def execInspection(data): try: inspection_id = int(data['inspection_id']) inspection_type = int(data['task_type']) site_id = str(data['site_id']) robots = data['robots'] robot_ids = [robot['robot_id'] for robot in robots] trigger = None if 'trigger_alarms' not in data.keys() else data['trigger_alarms'] ''' #assign robot model by split robot_id ''' for robot in robots: