def processPipe(self):
# Read data from pipe to worker thread, if available
#
while self.Sync.pipeCli.poll():
data = self.Sync.pipeCli.recv()
if data[0] == mpr.PipeValType.toCli_log:
# Handle log data -> write to history
#
self.log(data)
elif data[0] == mpr.PipeValType.toCli_displayInfo:
# Handle display information data -> update GUI
#
self.Stage = pickle._loads(data[1])
self.isLCrUsed = (self.Conf.useLCr
and (self.Stage.scrDevType
== stg.ScrDevType.DLPLCR4500EVM))
self.updateAll()
self.updateDisplayInfo()
elif data[0] == mpr.PipeValType.toCli_IODevInfo:
self.isIODevReady = data[1][0]
if self.isIODevReady == None:
self.isIODevReady = False
self.lastIOInfo = data[1]
self.updateIOInfo()
else:
# ***************************
# ***************************
# TODO: Other types of data need to be processed
# ***************************
# ***************************
pass
def get_function(self, function_bytes):
eggroll_serdes.bytes_security_check(function_bytes)
try:
return cloudpickle.loads(function_bytes)
except:
import pickle
return pickle._loads(function_bytes)
def _create_functor(self, task_info: processor_pb2.TaskInfo):
if task_info.function_bytes == b'blank':
return None
try:
return cloudpickle.loads(task_info.function_bytes)
except:
import pickle
return pickle._loads(task_info.function_bytes)
def loads(data):
data = to_bytes(data)
try:
return pickle.loads(data)
except:
if is_py3:
return pickle._loads(data)
raise
def loads(self, s, *, fix_imports=True, encoding="ASCII", errors="strict"):
if self._pickle_module == "P":
return pickle._loads(s,
fix_imports=fix_imports,
encoding=encoding,
errors=errors)
elif self._pickle_module == "C":
return pickle.loads(s,
fix_imports=fix_imports,
encoding=encoding,
errors=errors)
raise ValueError("Invalid pickle module")
def get_params_redis(shapes):
i = -1
params = []
keys = []
for s in range(len(shapes)):
keys.append(s)
values = asyncio.get_event_loop().run_until_complete(
multi_get_key_redis(keys))
for shape in shapes:
i = i + 1
param_np = pc._loads(values[i]).reshape(shape)
param_tensor = torch.nn.Parameter(torch.from_numpy(param_np))
params.append(param_tensor)
return params
def client_connection_thread(conn, addr):
# welcomes the new client
send_message(conn, b'Welcome to the Server, please login\n')
while True:
# tells client to send a command
send_message(conn, b'please input a command')
data = receive_message(conn)
# if nothing received then ends link
if not data:
break
# gets the data
data_split = pickle._loads(data)
data_split[0] = data_split[0].upper()
print(data_split[0])
if data_split[0] == 'DISCONNECT':
print('Connection from ', addr, ' has disconnected')
conn.close()
break
else:
server_commands(data_split, conn)
def fromFile(cls, filename):
"""
Load a suffix array instance from filename, a file created by
toFile.
Accept any filename following the _open conventions.
"""
self = cls.__new__(cls) #new instance which does not call __init__
start = _time()
savedData = _loads(_open(filename, "r").read())
# load common attributes
self.string, self.unit, self.voc, self.vocSize, self.SA, features = savedData[:
6]
self.length = len(self.SA)
# determine token delimiter
if self.unit == UNIT_WORD:
self.tokSep = " "
elif self.unit in (UNIT_CHARACTER, UNIT_BYTE):
self.tokSep = ""
else:
raise Exception("Unknown unit type identifier:", self.unit)
# recompute tokId based on voc
self.tokId = dict((char, iChar) for iChar, char in enumerate(self.voc))
self.nbSentences = self.string.count(self.tokId.get("\n", 0))
# Load features
self.features = []
for featureName, (featureValues,
featureDefault) in zip(features, savedData[6:]):
self.addFeatureSA((lambda _: featureValues),
name=featureName,
default=featureDefault)
self.fromFileTime = _time() - start
if _trace: print >> _stderr, "fromFileTime %.2fs" % self.fromFileTime
return self
def fromFile(cls, filename):
"""
Load a suffix array instance from filename, a file created by
toFile.
Accept any filename following the _open conventions.
"""
self = cls.__new__(cls) #new instance which does not call __init__
start=_time()
savedData=_loads(_open(filename, "r").read())
# load common attributes
self.string, self.unit, self.voc, self.vocSize, self.SA, features = savedData[:6]
self.length=len(self.SA)
# determine token delimiter
if self.unit==UNIT_WORD:
self.tokSep=" "
elif self.unit in (UNIT_CHARACTER, UNIT_BYTE):
self.tokSep=""
else:
raise Exception("Unknown unit type identifier:", self.unit)
# recompute tokId based on voc
self.tokId=dict((char, iChar) for iChar,char in enumerate(self.voc))
self.nbSentences = self.string.count(self.tokId.get("\n", 0))
# Load features
self.features=[]
for featureName, (featureValues, featureDefault) in zip(features, savedData[6:]):
self.addFeatureSA((lambda _: featureValues), name=featureName, default=featureDefault)
self.fromFileTime=_time()-start
if _trace: print >> _stderr, "fromFileTime %.2fs"%self.fromFileTime
return self
def load(self, name):
"""
Labeled images are stored on a persistent database with a unique name.
load() overwrites existing.
"""
obj, self.name, self.owner = LabeledImage.db_cursor.execute(
'SELECT I.Object,I.name,I.owner FROM LabeledImage I WHERE I.name =?',
(name, )).fetchone()
resolved_obj = pickle._loads(obj)
self.Image = resolved_obj['ImBytes'] # image is loaded
#next, get rules with same order as it was added
self.Rules = [None] * len(resolved_obj['Rules'])
for rule in resolved_obj['Rules']:
expr_type = rule['expr_type']
expr = rule['expr']
shape_type = rule['shape_type']
radius = rule['radius']
action = rule['action']
points = rule['points'] # another dictionary list
point_list = [None] * len(
points) # points also needed to save order
for point in points:
x1 = point['x1']
y1 = point['y1']
posp = point['position']
point_list[posp] = (x1, y1)
match_expr = MatchExpr(expr_type, expr)
if shape_type == Shape.CIRCLE:
shape = Shape(shape_type, radius, *point_list)
else:
shape = Shape(shape_type, *point_list)
pos = rule['position']
self.Rules[pos] = (match_expr, shape, action)
return self
def Main():
while True:
host = socket.gethostname() # Get local machine name
port = 12345 # Reserve a port for your service.
s = socket.socket() # Create a socket object
s.bind((host, port)) # Bind to the port
s.listen(5) # Now wait for client connection.
c, addr = s.accept() # Establish connection with client.
print('Got connection from' + str(addr))
while True:
data = c.recv(1024)
if not data:
break
commands = pickle._loads(data)
if commands[0].upper() == 'DISCONNECT':
c.close()
break
else:
serverCommands(commands)
c.close() # Close the connection
def update_event(self, inp=-1):
self.set_output_val(0, pickle._loads(self.input(0)))
def read(self, buffer):
return pickle._loads(buffer)
def get_function(self, function_bytes):
try:
return cloudpickle.loads(function_bytes)
except:
import pickle
return pickle._loads(function_bytes)
def roundtrip(val):
pickled = pickle_function.dumps(val)
# use _loads (the python implementation) for better stacktraces
return pickle._loads(pickled)
def unwrap(data):
message = pickle._loads(data)
return message
def upload_events_from_pickle_to_sql(project='tvcbook', remark='production'):
#所有的文件
filelist = []
dirpath = os.path.join('data_export', project, remark, 'events')
for maindir, subdir, file_name_list in os.walk(dirpath):
# print("1:",maindir) #当前主目录
# print("2:",subdir) #当前主目录下的所有目录
# print("3:",str(file_name_list)) #当前主目录下的所有文件
# file_name_list.sort()
# subdir.sort()
for filename in file_name_list:
apath = os.path.join(maindir, filename) #合并成一个完整路径
filelist.append(apath)
# print(file_name_list)
filelist.sort()
# print(filelist)
for pkl in filelist:
# print(pkl)
with open(pkl, "rb") as f2:
results = pickle._loads(f2.read())
# p = multiprocessing.Pool(processes = 3)
for item in results:
# # print(item)
try:
itemdict = json.loads(item)
all_json = {
"properties": itemdict,
"distinct_id": itemdict["distinct_id"],
"event": itemdict["event"],
"type": "track"
}
# first_id = itemdict['first_id'] if 'first_id' in itemdict else None
# second_id = itemdict['second_id'] if 'second_id' in itemdict else None
# unionid = itemdict['unionid'] if 'unionid' in itemdict else None
# id = itemdict['id'] if 'id' in itemdict else None
ip_city, ip_is_good = get_addr(itemdict["$ip"])
ip_asn, ip_asn_is_good = get_asn(itemdict["$ip"])
if ip_is_good == 0:
ip_city = '{}'
if ip_asn_is_good == 0:
ip_asn = '{}'
print(all_json)
created_at = time.mktime(
time.strptime(itemdict["time"].split('.')[0],
'%Y-%m-%d %H:%M:%S'))
# all_json = json.dumps(itemdict,ensure_ascii=False)
insert_data(project='tvcbook',
data_decode=all_json,
User_Agent=None,
Host=None,
Connection=None,
Pragma=None,
Cache_Control=None,
Accept=None,
Accept_Encoding=None,
Accept_Language=None,
ip=itemdict["$ip"] if "$ip" in itemdict else None,
ip_city=ip_city,
ip_asn=ip_asn,
url=None,
referrer=itemdict["$referrer"]
if "$referrer" in itemdict else None,
remark=remark,
ua_platform=itemdict["$lib"]
if "$lib" in itemdict else None,
ua_browser=itemdict["$browser"]
if "$browser" in itemdict else None,
ua_version=itemdict["$browser_version"]
if "$browser_version" in itemdict else None,
ua_language=None,
ip_is_good=ip_is_good,
ip_asn_is_good=ip_asn_is_good,
created_at=created_at)
except Exception:
error = traceback.format_exc()
write_to_log(filename='import_from_sa',
defname='upload_events_from_pickle_to_sql',
result=error)
# f2.close()
# p.apply_async(func=insert_data,kwds={
# "project":"tvcbook",
# "data_decode":all_json,
# "User_Agent":None,
# "Host":None,
# "Connection":None,
# "Pragma":None,
# "Cache_Control":None,
# "Accept":None,
# "Accept_Encoding":None,
# "Accept_Language":None,
# "ip":itemdict["$ip"] if "$ip" in itemdict else None,
# "ip_city":ip_city,
# "ip_asn":ip_asn,
# "url":None,
# "referrer":itemdict["$referrer"] if "$referrer" in itemdict else None,
# "remark":'production',
# "ua_platform":itemdict["$lib"] if "$lib" in itemdict else None,
# "ua_browser":itemdict["$browser"] if "$browser" in itemdict else None,
# "ua_version":itemdict["$browser_version"] if "$browser_version" in itemdict else None,
# "ua_language":None,
# "ip_is_good":ip_is_good,
# "ip_asn_is_good":ip_asn_is_good,
# "created_at":created_at})
# # insert_data
# p.close()
# p.join()
os.remove(pkl)
def upload_users_from_pickle_to_sql(project='tvcbook', remark='production'):
dirpath = os.path.join('data_export', project, remark, 'users')
filepath = os.path.join(dirpath, 'users_all.pkl')
with open(filepath, "rb") as f2:
results = pickle._loads(f2.read())
a = 1
for item in results:
print(a)
a += 1
# print('a',item)
data_rebuild = {
"properties": {},
"lib": {},
"distinct_id": "",
"event": "",
"type": "profile_set"
}
# print(item)
try:
item = json.loads(item)
# print(item["first_id"])
if len(item["first_id"]) == 16:
data_rebuild['lib']['$lib'] = 'js'
elif len(item['first_id']) >= 39 and len(item['first_id']) <= 46:
data_rebuild['lib']['$lib'] = 'MiniProgram'
elif len(item['first_id']) >= 51 and len(item['first_id']) <= 64:
data_rebuild['lib']['$lib'] = 'js'
else:
data_rebuild['lib']['$lib'] = 'unknow'
if 'second_id' in item:
data_rebuild["distinct_id"] = item['second_id']
data_rebuild["map_id"] = item['first_id']
data_rebuild["original_id"] = item['first_id']
if 'userid' in item:
# data_rebuild["properties"]["user_id"] = item['userid']
data_rebuild["properties"]["userId"] = item['userid']
if 'name' in item:
data_rebuild["properties"]["name"] = item['name']
if 'realname' in item:
data_rebuild["properties"]["realname"] = item['realname']
if 'sex' in item:
data_rebuild["properties"]["sex"] = item['sex']
if 'verification_type' in item:
data_rebuild["properties"]["verification_type"] = item[
'verification_type']
if 'company' in item:
data_rebuild["properties"]["company"] = item['company']
# print(item)
else:
data_rebuild["distinct_id"] = item['first_id']
# data_rebuild["map_id"] = item['first_id']
# data_rebuild["original_id"] = item['first_id']
if 'userid' in item:
# data_rebuild["properties"]["user_id"] = item['userid']
data_rebuild["properties"]["userId"] = item['userid']
if 'name' in item:
data_rebuild["properties"]["name"] = item['name']
if 'realname' in item:
data_rebuild["properties"]["realname"] = item['realname']
if 'sex' in item:
data_rebuild["properties"]["sex"] = item['sex']
if 'verification_type' in item:
data_rebuild["properties"]["verification_type"] = item[
'verification_type']
if 'company' in item:
data_rebuild["properties"]["company"] = item['company']
if 'viptype' in item:
data_rebuild["properties"]["viptype"] = item['viptype']
print(data_rebuild)
insert_user(project='tvcbook',
data_decode=data_rebuild,
created_at=0)
except Exception:
error = traceback.format_exc()
write_to_log(filename='import_from_sa',
defname='upload_users_from_pickle_to_sql',
result=error)
os.remove(filepath)
def loads(s, **kwargs):
return _loads(s)
def loads(s):
if not isinstance(s, bytes):
s = s.encode(errors='ignore')
return pickle._loads(s)
ls = [
b'C6yOrzSFsfy4bQ172sS2PRmpTmGa8euo+xg',
b'rTDAVInfyDn+WO72sS2PRmpTmGykx74Kz/HC4='
]
ls.append(str(keyobj)[2:-1])
jsondict = {'pubkey': ls[2], 'data': ls[0], 'key': ls[1]}
#
jd = js.dumps(jsondict)
ds = js.loads(jd)
vpubkey = bytes(ds['pubkey'], 'utf-8')
vpubkeyORIGINAL = vpubkey.decode('unicode-escape').encode('ISO-8859-1')
print(vpubkeyORIGINAL)
VOTER_PUBLIC_KEY = pk._loads(vpubkeyORIGINAL)
print(VOTER_PUBLIC_KEY)
print(VOTER_PUBLIC_KEY == keys['public'])
# jsondict['pubkey'] = str(keyobj)
# #print(jsondict)
#
# news = jsondict['pubkey']
# newnew = news[2:-1]
# b = bytes(newnew,'utf-8')
# print()
# newbie = b.decode('unicode-escape').encode('ISO-8859-1')
# print(newbie)
#
# ch = pk._loads(newbie)
#
def test_pickle(self):
logger = TestLogger(log_level=logging.ERROR)
logger = pickle._loads(pickle._dumps(logger))
self.assertIsInstance(logger._log, logging.Logger)
self.assertEqual(logger._log.level, logging.ERROR)
def deserialize_data(data):
return pickle._loads(data)
def loads(s, **kwargs): return _loads(s)
import pickle
import pprint
# pprint모둘은 리스트를 보기좋게 출력해준다
# pickle은 파이선 객체를 bytes타입으로 직렬화를 처리하는 모듈
data = [{
'name': '한사람',
'age': 27
}, {
'name': '두사람',
'age': 33
}, {
'name': '세사람',
'age': 18
}]
print(data)
pprint.pprint(data)
data_string = pickle.dumps(data)
print("pickle : ", data_string)
load_string = pickle._loads(data_string)
print('읽기 : ', load_string)
