def __init__(self, queue_size=128):
self.stopped = False
self.config = Config()
self.packer = Packer()
self.init_config()
# self.Q = PriorityQueue(maxsize=self.queue_size)
self.Q = Queue(maxsize=self.queue_size)
self.img_Q = Queue(maxsize=self.queue_size)
self.piece_array = []
self.piece_time = int(time.time() * 1000)
self.piece_fps = 40
for i in range(self.packer.frame_pieces):
self.piece_array.append(None)
# init timestamp
self.frame = numpy.zeros(self.packer.frame_size_3d, dtype=numpy.uint8)
self.imshow = self.frame.reshape(self.packer.h, self.packer.w,
self.packer.d)
self.last_frame_time = int(time.time() * 1000)
self.require = True
self.time_delay = 0
self.delay_timer = int(time.time() * 1000)
self.lock = Lock()
def unpack(self, packet):
Packer.unpack(self, packet)
for i, j in self.dict.items():
url = Url(-1, '')
url.unpack(j)
self.dict[i] = url
def parse(path, f=None):
p = Parser(path=path)
p.parse_file()
j = Packer(parser=p)
if f is None:
return j.pack()
else:
j.pack(f=f)
def parse(path, f=None):
p = Parser(path=path)
p.parse_file()
j = Packer(parser=p)
if f is None:
return j.pack()
else:
j.pack(f=f)
def __init__(self, client, filenames, key):
super().__init__(filenames)
self._logger = logging.getLogger('DropboxBackup')
self._packer = Packer(key)
self._client = client
for f in self._filenames:
if not os.path.exists(f):
self.restore(f)
def parse(path, f=None):
p = Parser(path=path)
p.parse_file()
a = Analyzer(parser=p)
a.analyze()
j = Packer(analyzer=a)
if f is None:
return j.pack()
else:
j.pack(f=f)
def processFile(path, f=None):
p = Parser(path=path)
p.parseFile()
a = Analyzer(parser=p)
a.analyze()
j = Packer(analyzer=a)
if f is None:
return j.pack()
else:
j.pack(f=f)
def processFile(path):
try:
p = Parser(path=path)
p.parseFile()
a = Analyzer(parser=p)
a.analyze()
j = Packer(analyzer=a)
return j.pack()
except:
print path
exit(1)
def processFile(path):
try:
p = Parser(path=path)
p.parseFile()
a = Analyzer(parser=p)
a.analyze()
j = Packer(analyzer=a)
return j.pack()
except:
print path
exit(1)
def test_get_export_data(self):
settings = {
'settings': {
'unit.test': {
'domain': 'unit.test',
'extras': '#!"§$%&/()[]{}=-_+*<>;:.',
'passwordTemplate': 'xnxoaAxxxx',
'iterations': 5000,
'notes': 'Nice note!',
'salt': 'cGVwcGVy',
'cDate': '2011-02-12T11:07:31',
'mDate': '2011-02-12T11:07:32'
},
'some.domain': {
'domain': 'some.domain',
'extras': '6478593021',
'passwordTemplate': 'xnxoaA',
'iterations': 4096,
'salt': 'cGVwcGVy',
'cDate': '2013-06-17T04:03:41',
'mDate': '2014-08-02T10:37:12'
}
},
'synced': []
}
salt = os.urandom(32)
kgk_manager = KgkManager()
kgk_manager.set_preference_manager(self.preference_manager)
kgk_manager.create_new_kgk()
kgk_block = kgk_manager.create_and_save_new_kgk_block(
Crypter(Crypter.createIvKey(b'xyz', salt, iterations=3)))
crypter = PasswordSettingsManager.get_settings_crypter(kgk_manager)
f = open(os.path.expanduser('~/.ctSESAM_test.pws'), 'bw')
f.write(salt + kgk_block + crypter.encrypt(
struct.pack('!I', 0) +
Packer.compress(json.dumps(settings).encode('utf-8'))))
f.close()
self.preference_manager.read_file()
self.manager.load_local_settings(kgk_manager)
data = b64decode(self.manager.get_export_data(kgk_manager))
self.assertEqual(b'\x01', data[:1])
salt = data[1:33]
kgk_crypter = Crypter(Crypter.createIvKey(b'xyz', salt, iterations=3))
kgk_manager2 = KgkManager()
kgk_manager2.set_preference_manager(self.preference_manager)
kgk_manager2.decrypt_kgk(data[33:145], kgk_crypter)
settings_crypter = PasswordSettingsManager.get_settings_crypter(
kgk_manager2)
self.assertEqual(
settings['settings'],
json.loads(
str(Packer.decompress(settings_crypter.decrypt(data[145:])),
encoding='utf-8')))
def __init__(self, queue_size=128):
self.stopped = False
self.config = Config()
self.packer = Packer()
self.init_config()
# self.Q = PriorityQueue(maxsize=self.queue_size)
self.Q = Queue(maxsize=self.queue_size)
self.img_Q = Queue(maxsize=self.queue_size)
# init timestamp
self.frame = numpy.zeros(self.packer.frame_size_3d, dtype=numpy.uint8)
self.last_frame_time = int(time.time() * 1000)
self.require = True
def __init__(self, src="C:\\Tools\\titan_test.mp4"):
self.config = Config()
self.packer = Packer()
# initialize the file video stream along with the boolean
# used to indicate if the thread should be stopped or not
os.environ["OPENCV_VIDEOIO_DEBUG"] = "1"
os.environ["OPENCV_VIDEOIO_PRIORITY_MSMF"] = "0"
encode_param = [int(cv2.IMWRITE_JPEG_QUALITY), 15]
self.stream = cv2.VideoCapture(src)
# while True:
# if cv2.waitKey(1) & 0xFF == ord('q'):
# break
# ret, frame = self.stream.read()
# if ret:
# # print(frame.shape)
# # frame = frame.reshape(self.packer.h, self.packer.w, self.packer.d)
# cv2.imshow('read video data.jpg', frame)
# self.stream.set(cv2.CAP_PROP_MODE, cv2.CAP_MODE_YUYV)
# print(self.stream.get(cv2.CAP_PROP_FPS)) # 默认帧率30
# self.stream.set(cv2.CAP_PROP_FPS, 20) # cv version is 3.4.2
self.stopped = False
self.requesting = False
self.request = False
self.quit = False
self.fps = 40
self.push_sleep = 0.01
self.push_sleep_min = 0.001
self.push_sleep_max = 0.2
self.piece_array = []
self.piece_time = int(time.time() * 1000)
self.piece_fps = 40
for i in range(self.packer.frame_pieces):
self.piece_array.append(None)
self.frame = numpy.zeros(self.packer.frame_size_3d, dtype=numpy.uint8)
self.imshow = self.frame.reshape(self.packer.h, self.packer.w,
self.packer.d)
self.frame_size = 0
self.piece_size = 0
self.frame_pieces = 0
self.init_config()
self.init_connection()
# intialize thread and lock
self.thread = Thread(target=self.update, args=())
self.thread.daemon = True
def get_export_data(self, kgk_manager):
"""
This gives you a base64 encoded string of encrypted settings data (the blob).
:param kgk_manager: kgk manager
:type kgk_manager: KgkManager
:return: encrypted settings blob
:rtype: str
"""
kgk_block = kgk_manager.get_fresh_encrypted_kgk()
settings_list = self.get_settings_as_dict()['settings']
if self.remote_data:
for domain_name in self.remote_data.keys():
data_set = self.remote_data[domain_name]
if 'deleted' in data_set and data_set['deleted']:
for i, setting_dict in enumerate(settings_list):
if setting_dict['domain'] == setting_dict['domain'] and datetime.strptime(
data_set['mDate'], "%Y-%m-%dT%H:%M:%S") > datetime.strptime(
setting_dict['mDate'], "%Y-%m-%dT%H:%M:%S"):
settings_list[i] = data_set
if domain_name not in settings_list.keys():
settings_list[domain_name] = {
'mDate': datetime.now(),
'deleted': True
}
settings_crypter = self.get_settings_crypter(kgk_manager)
return b64encode(b'\x01' + kgk_manager.get_kgk_crypter_salt() + kgk_block + settings_crypter.encrypt(
Packer.compress(json.dumps(settings_list))))
def test_store_local_settings(self):
abc_setting = self.manager.get_setting('abc.de')
abc_setting.set_template('xAxonaxxxx')
self.manager.set_setting(abc_setting)
new_setting = PasswordSetting('hugo.com')
new_setting.set_template('xonxAxxaxxxx')
self.manager.set_setting(new_setting)
kgk_manager = KgkManager()
kgk_manager.set_preference_manager(self.preference_manager)
kgk_manager.create_new_kgk()
salt = os.urandom(32)
kgk_manager.create_and_save_new_kgk_block(
Crypter(Crypter.createIvKey(b'xyz', salt, iterations=3)))
self.manager.store_local_settings(kgk_manager)
with open(os.path.expanduser('~/.ctSESAM_test.pws'), 'br') as f:
data = f.read()
settings_crypter = PasswordSettingsManager.get_settings_crypter(
kgk_manager)
decrypted_settings = settings_crypter.decrypt(data[144:])
sync_settings_len = struct.unpack('!I', decrypted_settings[:4])[0]
data = json.loads(
Packer.decompress(
decrypted_settings[4 + sync_settings_len:]).decode('utf8'))
self.assertEqual('abc.de', data['settings']['abc.de']['domain'])
self.assertEqual('xAxonaxxxx',
data['settings']['abc.de']['passwordTemplate'])
self.assertEqual('hugo.com', data['settings']['hugo.com']['domain'])
self.assertEqual('xonxAxxaxxxx',
data['settings']['hugo.com']['passwordTemplate'])
def get_export_data(self, kgk_manager):
"""
This gives you a base64 encoded string of encrypted settings data (the blob).
:param kgk_manager: kgk manager
:type kgk_manager: KgkManager
:return: encrypted settings blob
:rtype: str
"""
kgk_block = kgk_manager.get_fresh_encrypted_kgk()
settings_list = self.get_settings_as_dict()['settings']
if self.remote_data:
for domain_name in self.remote_data.keys():
data_set = self.remote_data[domain_name]
if 'deleted' in data_set and data_set['deleted']:
for i, setting_dict in enumerate(settings_list):
if setting_dict['domain'] == setting_dict['domain'] and \
PasswordSetting.convert_ISO_date(data_set['mDate']) > \
PasswordSetting.convert_ISO_date(setting_dict['mDate']):
settings_list[i] = data_set
if domain_name not in settings_list.keys():
settings_list[domain_name] = {
'mDate': datetime.now(),
'deleted': True
}
settings_crypter = self.get_settings_crypter(kgk_manager)
return b64encode(b'\x01' + kgk_manager.get_kgk_crypter_salt() +
kgk_block + settings_crypter.encrypt(
Packer.compress(json.dumps(settings_list))))
def unpack(self, packet):
Packer.unpack(self, packet)
for i, j in self.progresses.items():
prog = Progress(self, -1, -1, -1)
prog.unpack(j)
self.progresses[i] = prog
self.makeMap()
for progress in self.progresses.values():
pos = int(progress.begin * 1.0 / self.handler.file.BLOCK_SIZE)
fil_len = int(math.ceil(progress.go_inc * 1.0 / self.handler.file.BLOCK_SIZE))
self.block_map[pos] = progress.urlid
for i in range(fil_len):
self.block_map[pos + i] = progress.urlid
self.status.endflag = self.status.go_end = self.status.done_end = self.isEnd()
def update_from_export_data(self, kgk_manager, blob):
"""
Call this method to pull settings from the sync server.
:param kgk_manager: the kgk manager used for the decryption
:type kgk_manager: KgkManager
:param blob: the export data
:type blob: bytes
"""
if not blob[0] == 1:
print("Version error: Wrong data format. Could not import anything.")
return True
settings_crypter = self.get_settings_crypter(kgk_manager)
decrypted_settings = settings_crypter.decrypt(blob[145:])
if len(decrypted_settings) <= 0:
print("Wrong password.")
return False
self.remote_data = json.loads(str(Packer.decompress(decrypted_settings), encoding='utf-8'))
self.update_remote = False
for domain_name in self.remote_data.keys():
data_set = self.remote_data[domain_name]
found = False
i = 0
while i < len(self.settings):
setting = self.settings[i]
if setting.get_domain() == domain_name:
found = True
if datetime.strptime(data_set['mDate'], "%Y-%m-%dT%H:%M:%S") > setting.get_m_date():
if 'deleted' in data_set and data_set['deleted']:
self.settings.pop(i)
else:
setting.load_from_dict(data_set)
setting.set_synced(True)
self.update_remote = True
i += 1
else:
i += 1
else:
i += 1
if not found:
new_setting = PasswordSetting(domain_name)
new_setting.load_from_dict(data_set)
new_setting.set_synced(True)
self.settings.append(new_setting)
for setting in self.settings:
found = False
for domain_name in self.remote_data.keys():
data_set = self.remote_data[domain_name]
if setting.get_domain() == domain_name:
found = True
if setting.get_m_date() >= datetime.strptime(data_set['mDate'], "%Y-%m-%dT%H:%M:%S"):
self.update_remote = True
if not found:
self.update_remote = True
self.store_local_settings(kgk_manager)
return self.update_remote
def collate(self, batch, is_train=True):
if is_train:
packer = Packer([x['teller_tokens_in'] for x in batch])
return {
'packer':
packer,
'brw_teller_tokens_in':
packer.brw_from_list([x['teller_tokens_in']
for x in batch]).to(cuda_if_available),
'brw_teller_counts_in':
packer.brw_from_list([x['teller_counts_in']
for x in batch]).to(cuda_if_available),
'brw_teller_tokens_out':
packer.brw_from_list([x['teller_tokens_out']
for x in batch]).to(cuda_if_available),
'b_scene_tags':
torch.stack([x['scene_tags']
for x in batch]).to(cuda_if_available),
'b_scene_mask':
torch.stack([x['scene_mask']
for x in batch]).to(cuda_if_available),
'br_drawer_clipart_state':
packer.br_from_list([x['drawer_clipart_state']
for x in batch]).to(cuda_if_available),
}
else:
return {
'b_scene_tags':
torch.stack([x['scene_tags']
for x in batch]).to(cuda_if_available),
'b_scene_mask':
torch.stack([x['scene_mask']
for x in batch]).to(cuda_if_available),
}
def main():
examples = []
examples.append(("-n test.cfg", "creates empty config file"))
examples.append(("-v sprite.cfg", "runs sprite.cfg with messages"))
ex_str = "\n".join([" %prog {0:<18}{1}".format(tup[0], tup[1]) for tup in examples])
usage = "Usage: %prog [options] <config file> [<config file> ...]\n\nExamples:\n{0}".format(ex_str)
parser = OptionParser(usage=usage, version="%prog {0}".format(Packer.version_str()))
parser.add_option("-v", "--verbose", action="store_true", help="print messages to stdout", dest="verbose")
parser.add_option("-n", "--new-config", action="store_true", help="creates an empty config file; recommended way of creating a new sprite.", dest="newcfg")
#parser.add_option("--hd", action="store_true", help="works only with -n, creates an empty config file perfect for generating ", dest="create-cfg")
(options, args) = parser.parse_args()
if args:
for arg in args:
packer = Packer(arg)
if options.newcfg:
packer.create_cfg(arg)
else:
for etype_and_msg in packer.run_with_messages():
etype, msg = etype_and_msg
if etype == ERROR:
sys.stderr.write(msg + "\n")
elif etype == MESSAGE and options.verbose:
print msg
if packer.error_msg():
sys.exit(1)
def _pack_test(self):
if len(self.textures) == 0:
return
from packer import PackingRectangle, Packer
# prepare rects
rects = []
rects2 = {}
for texture in self.textures:
r = PackingRectangle()
r.size = texture.get("img").size
rects.append(r)
rects2[r] = (texture.get("name"), texture.get("img"))
# pack
packer = Packer(rects)
res, area = packer.pack()
uvs = {}
w = 0
h = 0
for e in res:
w = max(w, e.right)
h = max(h, e.bottom)
for e in res:
rect = e.rect
uvs[rects2.get(e)[0]] = (rect[0] / float(w), rect[1] / float(h), rect[2] / float(w), rect[3] / float(h)) # u1, v1, u2, v2
print("packed into", w, h)
# generate new image
img = Image.new('RGBA', (w, h))
for e in res:
img.paste(rects2.get(e)[1], e.position)
self.textures = [{"path": "atlas.png", "name": "texture_atlas", "index": 0, "img": img}]
for spr in self.sprites:
uv = uvs.get(spr["texture_name"])
spr["uv"] = {"u1": float(uv[0]), "v1": float(uv[1]), "u2": float(uv[2]), "v2": float(uv[3])}
spr["texture_name"] = "texture_atlas"
def test_load_settings_from_file(self):
settings = {
'settings': {
'unit.test': {
'domain': 'unit.test',
'passwordTemplate': 'xxxxxxxxxxo',
'extras': '#OWspx6;3gov0/1',
'iterations': 5000,
'notes': 'Nice note!',
'cDate': '2011-02-12T11:07:31',
'mDate': '2011-02-12T11:07:32'
},
'some.domain': {
'domain': 'some.domain',
'passwordTemplate': 'oxxx',
'extras': '6478593021',
'cDate': '2013-06-17T04:03:41',
'mDate': '2014-08-02T10:37:12'
}
},
'synced': []
}
salt = os.urandom(32)
data = json.dumps(settings).encode('utf-8')
kgk_manager = KgkManager()
kgk_manager.set_preference_manager(self.preference_manager)
kgk_manager.create_new_kgk()
kgk_block = kgk_manager.create_and_save_new_kgk_block(
Crypter(Crypter.createIvKey(b'xyz', salt, iterations=3)))
crypter = PasswordSettingsManager.get_settings_crypter(kgk_manager)
f = open(os.path.expanduser('~/.ctSESAM_test.pws'), 'bw')
f.write(salt + kgk_block +
crypter.encrypt(struct.pack('!I', 0) + Packer.compress(data)))
f.close()
self.preference_manager.read_file()
self.manager.load_local_settings(kgk_manager)
self.assertIn('unit.test', self.manager.get_domain_list())
self.assertIn('some.domain', self.manager.get_domain_list())
self.assertEqual('xxxxxxxxxxo',
self.manager.get_setting('unit.test').get_template())
self.assertEqual(
5000,
self.manager.get_setting('unit.test').get_iterations())
self.assertEqual('Nice note!',
self.manager.get_setting('unit.test').get_notes())
self.assertEqual(
'oxxx',
self.manager.get_setting('some.domain').get_template())
self.assertEqual(
'6478593021',
self.manager.get_setting('some.domain').get_character_set())
def test_get_domain_list(self):
settings = {
'settings': {
'unit.test': {
'domain': 'unit.test',
'extras': '#!"§$%&/()[]{}=-_+*<>;:.',
'passwordTemplate': 'xxxaoxxAxxn',
'iterations': 5000,
'notes': 'Nice note!',
'salt': 'cGVwcGVy',
'cDate': '2011-02-12T11:07:31',
'mDate': '2011-02-12T11:07:32'
},
'some.domain': {
'domain': 'some.domain',
'extras': '#!"§$%&/()[]{}=-_+*<>;:.',
'passwordTemplate': 'xxxo',
'iterations': 4096,
'salt': 'cGVwcGVy',
'cDate': '2013-06-17T04:03:41',
'mDate': '2014-08-02T10:37:12'
}
},
'synced': []
}
salt = os.urandom(32)
f = open(os.path.expanduser('~/.ctSESAM_test.pws'), 'bw')
data = json.dumps(settings).encode('utf-8')
kgk_manager = KgkManager()
kgk_manager.set_preference_manager(self.preference_manager)
kgk_manager.create_new_kgk()
kgk_block = kgk_manager.create_and_save_new_kgk_block(
Crypter(Crypter.createIvKey(b'xyz', salt, iterations=3)))
crypter = PasswordSettingsManager.get_settings_crypter(kgk_manager)
f.write(salt + kgk_block +
crypter.encrypt(struct.pack('!I', 0) + Packer.compress(data)))
f.close()
self.preference_manager.read_file()
self.manager.load_local_settings(kgk_manager)
self.assertIn('settings', self.manager.get_settings_as_dict())
self.assertIn('unit.test',
self.manager.get_settings_as_dict()['settings'])
self.assertEqual(
settings['settings']['unit.test'],
self.manager.get_settings_as_dict()['settings']['unit.test'])
self.assertIn('some.domain',
self.manager.get_settings_as_dict()['settings'])
self.assertEqual(
settings['settings']['some.domain'],
self.manager.get_settings_as_dict()['settings']['some.domain'])
self.assertEqual(settings, self.manager.get_settings_as_dict())
def store_local_settings(self, kgk_manager):
"""
This actually saves the settings to a file on the disk. The file is encrypted so you need to supply the
password.
:param kgk_manager: kgk manager
:type kgk_manager: KgkManager
"""
kgk_manager.fresh_salt2()
kgk_manager.fresh_iv2()
settings_crypter = PasswordSettingsManager.get_settings_crypter(kgk_manager)
sync_settings = self.sync_manager.get_binary_sync_settings()
self.preference_manager.store_settings_data(settings_crypter.encrypt(
struct.pack('!I', len(sync_settings)) + sync_settings +
Packer.compress(json.dumps(self.get_settings_as_dict()))))
kgk_manager.store_local_kgk_block()
def store_local_settings(self, kgk_manager):
"""
This actually saves the settings to a file on the disk. The file is encrypted so you need to supply the
password.
:param kgk_manager: kgk manager
:type kgk_manager: KgkManager
"""
kgk_manager.fresh_salt2()
kgk_manager.fresh_iv2()
settings_crypter = PasswordSettingsManager.get_settings_crypter(kgk_manager)
sync_settings = self.sync_manager.get_binary_sync_settings()
self.preference_manager.store_settings_data(settings_crypter.encrypt(
struct.pack('!I', len(sync_settings)) + sync_settings +
Packer.compress(json.dumps(self.get_settings_as_dict()))))
kgk_manager.store_local_kgk_block()
def get_binary_sync_settings(self):
"""
returns packed sync settings
:return: binary settings
:rtype: bytes
"""
if self.sync:
return Packer.compress(json.dumps({
"server-address": self.server_address,
"username": self.username,
"password": self.password,
"certificate": self.certificate
}).encode('utf-8'))
else:
return b''
def load_local_settings(self, kgk_manager):
"""
This loads the saved settings. It is a good idea to call this method the minute you have a kgk manager.
:param kgk_manager: kgk manager
:type kgk_manager: KgkManager
"""
encrypted_settings = self.preference_manager.get_settings_data()
if len(encrypted_settings) < 40:
return
settings_crypter = PasswordSettingsManager.get_settings_crypter(
kgk_manager)
decrypted_settings = settings_crypter.decrypt(encrypted_settings)
sync_settings_len = struct.unpack('!I', decrypted_settings[0:4])[0]
if sync_settings_len > 0:
self.sync_manager.load_binary_sync_settings(
decrypted_settings[4:4 + sync_settings_len])
if len(decrypted_settings) < sync_settings_len + 44:
raise ValueError("The decrypted settings are too short.")
decompressed_settings = Packer.decompress(
decrypted_settings[4 + sync_settings_len:])
if len(decompressed_settings) <= 0:
raise PermissionError(
"Wrong password: The settings could not decompress.")
saved_settings = json.loads(
str(decompressed_settings, encoding='utf-8'))
for domain_name in saved_settings['settings'].keys():
data_set = saved_settings['settings'][domain_name]
found = False
i = 0
while i < len(self.settings):
setting = self.settings[i]
if setting.get_domain() == domain_name:
found = True
if datetime.strptime(
data_set['mDate'],
"%Y-%m-%dT%H:%M:%S") > setting.get_m_date():
setting.load_from_dict(data_set)
setting.set_synced(
setting.get_domain() in saved_settings['synced'])
i += 1
if not found:
new_setting = PasswordSetting(domain_name)
new_setting.load_from_dict(data_set)
new_setting.set_synced(
new_setting.get_domain() in saved_settings['synced'])
self.settings.append(new_setting)
def pull(self):
"""
Returns some mock data tor the sync test.
:return: base64 mock data blob
:rtype: (bool, str)
"""
remote_data = {
'unit.test': {
'domain': 'unit.test',
'length': 12,
'iterations': 5001,
'notes': 'another note!',
'salt': 'cGVwcGVy',
'usedCharacters':
'abcdefghijklmnopqrstuvwxyzABCDEFGHJKLMNPQRTUVWXYZ0123456789',
'cDate': '2011-02-12T11:07:31',
'mDate': '2013-07-12T14:46:11'
},
'some.domain': {
'domain': 'some.domain',
'length': 4,
'iterations': 4097,
'salt': 'cGVwcGVy',
'usedCharacters': '6478593021',
'cDate': '2013-06-17T04:03:41',
'mDate': '2014-08-02T10:37:11'
},
'third.domain': {
'domain': 'third.domain',
'length': 10,
'iterations': 4098,
'salt': 'cGVwcGVy',
'usedCharacters': 'aeiou',
'cDate': '2013-06-17T04:03:41',
'mDate': '2014-08-02T10:37:11'
}
}
salt = os.urandom(32)
kgk_block = self.kgk_manager.create_and_save_new_kgk_block(
self.kgk_manager.get_kgk_crypter(b'xyz', salt))
settings_crypter = PasswordSettingsManager.get_settings_crypter(
self.kgk_manager)
return True, str(
b64encode(b'\x01' + salt + kgk_block + settings_crypter.encrypt(
Packer.compress(json.dumps(remote_data).encode('utf-8')))),
encoding='utf-8')
def __call__(self, ar): p = Packer(ar) for feature, size in self.features: p.addRoom(size) p.pack() for room, (feature, size) in zip(p.rooms, self.features): r = area.Area(room.left, room.top, room.right, room.bottom) for cell in feature(r): yield cell
def load_binary_sync_settings(self, data):
"""
loads sync settings
:param bytes data: packed json data of sync settings
"""
settings_dict = json.loads(
str(Packer.decompress(data), encoding='utf-8'))
if "server-address" in settings_dict and \
"username" in settings_dict and \
"password" in settings_dict:
self.server_address = settings_dict["server-address"]
self.username = settings_dict["username"]
self.password = settings_dict["password"]
self.certificate = None
if "certificate" in settings_dict:
self.certificate = settings_dict["certificate"] if len(
settings_dict["certificate"]) > 0 else None
self.create_certificate_file_if_needed()
self.create_sync()
def main():
examples = []
examples.append(("-n test.cfg", "creates empty config file"))
examples.append(("-v sprite.cfg", "runs sprite.cfg with messages"))
ex_str = "\n".join(
[" %prog {0:<18}{1}".format(tup[0], tup[1]) for tup in examples])
usage = "Usage: %prog [options] <config file> [<config file> ...]\n\nExamples:\n{0}".format(
ex_str)
parser = OptionParser(usage=usage,
version="%prog {0}".format(Packer.version_str()))
parser.add_option("-v",
"--verbose",
action="store_true",
help="print messages to stdout",
dest="verbose")
parser.add_option(
"-n",
"--new-config",
action="store_true",
help=
"creates an empty config file; recommended way of creating a new sprite.",
dest="newcfg")
#parser.add_option("--hd", action="store_true", help="works only with -n, creates an empty config file perfect for generating ", dest="create-cfg")
(options, args) = parser.parse_args()
if args:
for arg in args:
packer = Packer(arg)
if options.newcfg:
packer.create_cfg(arg)
else:
for etype_and_msg in packer.run_with_messages():
etype, msg = etype_and_msg
if etype == ERROR:
sys.stderr.write(msg + "\n")
elif etype == MESSAGE and options.verbose:
print msg
if packer.error_msg():
sys.exit(1)
def load_binary_sync_settings(self, data):
"""
loads sync settings
:param bytes data: packed json data of sync settings
"""
settings_dict = json.loads(str(Packer.decompress(data), encoding='utf-8'))
if "server-address" in settings_dict and \
"username" in settings_dict and \
"password" in settings_dict and \
"certificate" in settings_dict:
self.server_address = settings_dict["server-address"]
self.username = settings_dict["username"]
self.password = settings_dict["password"]
self.certificate = settings_dict["certificate"]
if self.certificate_file:
self.certificate_file.close()
self.certificate_file = NamedTemporaryFile()
self.certificate_file.write(self.certificate.encode('utf-8'))
self.certificate_file.seek(0)
self.create_sync()
def load_local_settings(self, kgk_manager):
"""
This loads the saved settings. It is a good idea to call this method the minute you have a kgk manager.
:param kgk_manager: kgk manager
:type kgk_manager: KgkManager
"""
encrypted_settings = self.preference_manager.get_settings_data()
if len(encrypted_settings) < 40:
return
settings_crypter = PasswordSettingsManager.get_settings_crypter(kgk_manager)
decrypted_settings = settings_crypter.decrypt(encrypted_settings)
sync_settings_len = struct.unpack('!I', decrypted_settings[0:4])[0]
if sync_settings_len > 0:
self.sync_manager.load_binary_sync_settings(decrypted_settings[4:4+sync_settings_len])
if len(decrypted_settings) < sync_settings_len+44:
raise ValueError("The decrypted settings are too short.")
decompressed_settings = Packer.decompress(decrypted_settings[4+sync_settings_len:])
if len(decompressed_settings) <= 0:
raise PermissionError("Wrong password: The settings could not decompress.")
saved_settings = json.loads(str(decompressed_settings, encoding='utf-8'))
for domain_name in saved_settings['settings'].keys():
data_set = saved_settings['settings'][domain_name]
found = False
i = 0
while i < len(self.settings):
setting = self.settings[i]
if setting.get_domain() == domain_name:
found = True
if datetime.strptime(data_set['mDate'], "%Y-%m-%dT%H:%M:%S") > setting.get_m_date():
setting.load_from_dict(data_set)
setting.set_synced(setting.get_domain() in saved_settings['synced'])
i += 1
if not found:
new_setting = PasswordSetting(domain_name)
new_setting.load_from_dict(data_set)
new_setting.set_synced(new_setting.get_domain() in saved_settings['synced'])
self.settings.append(new_setting)
class Game:
def __init__(self, account):
self.username = account['username']
self.password = account['password']
self.server = account['server']
self.shipClasses = { }
self.ships = { }
self.fleets = [ None, None, None, None ]
self.expeditions = { }
self.repairYards = [ None, None, None, None ]
self.maxShipNum = 0
self.errorCodes = { }
self.packer = Packer(self)
Log.i('Initializing game...')
self.conn = Connection()
self.gameData = self.conn.get('/index/getInitConfigs/')
for shipClassData in self.gameData['shipCard']:
if int(shipClassData['npc']) == 0 and int(shipClassData['release']) == 1:
shipClass = self.packer.makeShipClass(shipClassData)
self.shipClasses[shipClass.id] = shipClass
for expeditionData in self.gameData['pveExplore']:
expedition = self.packer.makeExpedition(expeditionData)
self.expeditions[expedition.id] = expedition
Log.errorCodes = { int(k) : v for k, v in self.gameData['errorCode'].items() }
Log.i('Logging in...')
loginData = self.conn.httpsGet('/index/passportLogin/%s/%s/' % (self.username, self.password))
self.conn.setServer(self.server)
time.sleep(1)
self.conn.get('//index/login/' + loginData['userId'])
Log.i('Initializing user data...')
self.userData = self.conn.get('/api/initGame/')
self.conn.get('/pevent/getPveData/')
self.conn.get('/pve/getPveData/')
time.sleep(5)
self.conn.get('/active/getUserData/')
self.conn.get('/campaign/getUserData/')
self.conn.get('/pve/getUserData/')
for shipData in self.userData['userShipVO']:
ship = self.packer.makeShip(shipData)
self.ships[ship.id] = ship
for fleetData in self.userData['fleetVo']:
fleet = self.packer.makeFleet(fleetData)
self.fleets[fleet.id - 1] = fleet
for expStatusData in self.userData['pveExploreVo']['levels']:
exp = self.expeditions[int(expStatusData['exploreId'])]
fleet = self.fleets[int(expStatusData['fleetId']) - 1]
endTime = datetime.fromtimestamp(int(expStatusData['endTime']))
exp.setStatus(fleet, endTime)
for repairYardData in self.userData['repairDockVo']:
ry = self.packer.makeRepairYard(repairYardData)
self.repairYards[ry.id - 1] = ry
self.maxShipNum = int(self.userData['userVo']['detailInfo']['shipNumTop'])
Log.i('Done')
def restart(self):
self.conn = Connection()
self.conn.get('/index/getInitConfigs/')
loginData = self.conn.get('/index/passportLogin/%s/%s/' % (self.username, self.password))
self.conn.setServer(self.server)
self.conn.get('//index/login/' + loginData['userId'])
self.conn.get('/api/initGame/')
Log.i('Game restarted')
def getShipClass(self, id_):
return self.shipClasses[id_]
def getShip(self, id_):
return self.ships[id_]
def findShip(self, name, lv = None):
ret = None
for ship in self.ships.values():
if ship.getName() == name:
if lv is not None and ship.lv == lv:
Log.i('Found ship %s of level %d, id: %d' % (name, ship.lv, ship.id))
return ship
if ret is None or ret.lv < ship.lv:
ret = ship
return ret
def getFleet(self, id_):
return self.fleets[id_ - 1]
def isDormFull(self):
return len(self.ships) >= self.maxShipNum
def addShip(self, ship):
self.ships[ship.id] = ship
def addResource(self, res):
pass # TODO
def removeShip(self, ship):
del self.ships[ship.id]
# Ship
def repair(self, ship, repairYard):
data = self.conn.get('/boat/repair/%d/%d/' % (ship.id, repairYard.id))
for ryData in data['repairDockVo']:
if int(ryData['id']) == repairYard.id:
return datetime.fromtimestamp(int(ryData['endTime']))
def repairComplete(self, repairYard):
self.conn.get('/boat/repairComplete/%d/%d/' % (repairYard.id, repairYard.ship.id))
def instantRepair(self, ship):
self.conn.get('/boat/instantRepairShips/[%d]/' % ship.id)
def dismantleShip(self, ship, keepEquipt = True):
self.conn.get('/dock/dismantleBoat/[%d]/%d/' % (ship.id, (0 if keepEquipt else 1)))
# Fleet
def changeShips(self, fleet, ships):
ships = [ ship.id for ship in ships ]
ships = str(ships).replace(' ', '')
self.conn.get('/boat/instantFleet/%d/%s/' % (fleet.id, ships))
def changeShip(self, fleet, pos, ship):
self.conn.get('/boat/changeBoat/%d/%d/%d/' % (fleet.id, ship.id, pos))
def fill(self, fleet):
ships = [ ship.id for ship in fleet.ships ]
ships = str(ships).replace(' ', '')
self.conn.get('/boat/supplyBoats/%s/' % ships)
# Expedition
def startExpedition(self, exp, fleet):
data = self.conn.post('/explore/start/%d/%d/' % (fleet.id, exp.id), 'pve_level=1')
for expData in data['pveExploreVo']['levels']:
if int(expData['exploreId']) == exp.id:
return datetime.fromtimestamp(int(expData['endTime']))
Log.e('startExpedition: unexpected response')
def getExpeditionResult(self, exp):
data = self.conn.get('/explore/getResult/%d/' % exp.id)
return (int(data['bigSuccess']) == 1), self.packer.makeResource(data['newAward'])
# Battle
def startStage(self, stageId, fleet):
self.conn.post('/pve/challenge/%d/%d/0/' % (stageId, fleet.id), 'pve_level=1')
def quitStage(self):
# faking official client
self.conn.get('/active/getUserData/')
self.conn.get('/pve/getUserData/')
self.conn.get('/campaign/getUserData/')
def nextSpot(self):
data = self.conn.get('/pve/next/')
return int(data['node'])
def searchEnemy(self, spot):
data = self.conn.get('/pve/spy/')
if int(data['enemyVO']['isFound']) == 0:
return [ ]
return [ int(ship['type']) for ship in data['enemyVO']['enemyShips'] ]
def startBattle(self, spot, fleet, formation):
data = self.conn.get('/pve/deal/%d/%d/%d/' % (spot, fleet.id, formation))
selfHp = data['warReport']['hpBeforeNightWarSelf']
enemyHp = data['warReport']['hpBeforeNightWarEnemy']
lastSpot = (int(data['pveLevelEnd']) == 1)
return selfHp, enemyHp, lastSpot
def getBattleResult(self, doNightBattle):
data = self.conn.get('/pve/getWarResult/%d/' % (1 if doNightBattle else 0))
newShip = None
if 'newShipVO' in data:
newShip = self.packer.makeShip(data['newShipVO'][0])
ships = None
if type(data['shipVO']) is list:
ships = [ self.packer.makeShip(shipData) for shipData in data['shipVO'] ]
else:
ships = [ self.packer.makeShip(data['shipVO']) ]
return newShip, ships
def __init__(self, account):
self.username = account['username']
self.password = account['password']
self.server = account['server']
self.shipClasses = { }
self.ships = { }
self.fleets = [ None, None, None, None ]
self.expeditions = { }
self.repairYards = [ None, None, None, None ]
self.maxShipNum = 0
self.errorCodes = { }
self.packer = Packer(self)
Log.i('Initializing game...')
self.conn = Connection()
self.gameData = self.conn.get('/index/getInitConfigs/')
for shipClassData in self.gameData['shipCard']:
if int(shipClassData['npc']) == 0 and int(shipClassData['release']) == 1:
shipClass = self.packer.makeShipClass(shipClassData)
self.shipClasses[shipClass.id] = shipClass
for expeditionData in self.gameData['pveExplore']:
expedition = self.packer.makeExpedition(expeditionData)
self.expeditions[expedition.id] = expedition
Log.errorCodes = { int(k) : v for k, v in self.gameData['errorCode'].items() }
Log.i('Logging in...')
loginData = self.conn.httpsGet('/index/passportLogin/%s/%s/' % (self.username, self.password))
self.conn.setServer(self.server)
time.sleep(1)
self.conn.get('//index/login/' + loginData['userId'])
Log.i('Initializing user data...')
self.userData = self.conn.get('/api/initGame/')
self.conn.get('/pevent/getPveData/')
self.conn.get('/pve/getPveData/')
time.sleep(5)
self.conn.get('/active/getUserData/')
self.conn.get('/campaign/getUserData/')
self.conn.get('/pve/getUserData/')
for shipData in self.userData['userShipVO']:
ship = self.packer.makeShip(shipData)
self.ships[ship.id] = ship
for fleetData in self.userData['fleetVo']:
fleet = self.packer.makeFleet(fleetData)
self.fleets[fleet.id - 1] = fleet
for expStatusData in self.userData['pveExploreVo']['levels']:
exp = self.expeditions[int(expStatusData['exploreId'])]
fleet = self.fleets[int(expStatusData['fleetId']) - 1]
endTime = datetime.fromtimestamp(int(expStatusData['endTime']))
exp.setStatus(fleet, endTime)
for repairYardData in self.userData['repairDockVo']:
ry = self.packer.makeRepairYard(repairYardData)
self.repairYards[ry.id - 1] = ry
self.maxShipNum = int(self.userData['userVo']['detailInfo']['shipNumTop'])
Log.i('Done')
def test_decompress(self):
self.assertEqual(
b'Some packable information',
Packer.decompress(b64decode("AAAAGXjaC87PTVUoSEzOTkzKSVXIzEvLL8pNLMnMzwMAedUJrg==")))
def test_compress(self):
packed_data = Packer.compress("Some packable information")
self.assertEqual(b'AAAAGXjaC87PTVUoSEzOTkzKSVXIzEvLL8pNLMnMzwMAedUJrg==', b64encode(packed_data))
from packer import Packer
from fdates import parse
P = Packer()
P.dataset(
1, {
"url": "https://www.betashares.com.au/files/nav/NDQ_History.csv",
"parse_as": "csv",
"name": "NAV History",
"index": "Date",
"subsets": ["NAV"],
"scrape_every": "1 day"
})
# toppling is the act of:
# 1. store index [0] of list as x
# 2. prepend x to list
# 3. pop last value off list
P.topple(1, "NAV")
P.dataset(
2, {
"url": "NDQ.AX.yfi",
"range": "daily",
"name": "NASDAQ ETF",
"index": "date",
"subsets": ["close"],
"scrape_every": "1 day"
})
class NetVideoStream:
def __init__(self, queue_size=128):
self.stopped = False
self.config = Config()
self.packer = Packer()
self.init_config()
# self.Q = PriorityQueue(maxsize=self.queue_size)
self.Q = Queue(maxsize=self.queue_size)
self.img_Q = Queue(maxsize=self.queue_size)
self.piece_array = []
self.piece_time = int(time.time() * 1000)
self.piece_fps = 40
for i in range(self.packer.frame_pieces):
self.piece_array.append(None)
# init timestamp
self.frame = numpy.zeros(self.packer.frame_size_3d, dtype=numpy.uint8)
self.imshow = self.frame.reshape(self.packer.h, self.packer.w,
self.packer.d)
self.last_frame_time = int(time.time() * 1000)
self.require = True
self.time_delay = 0
self.delay_timer = int(time.time() * 1000)
self.lock = Lock()
def init_config(self):
# 初始化大小信息
config = self.config
# 初始化连接信息
host = config.get("server", "host")
port = config.get("server", "port")
self.address = (host, int(port))
# 初始化打包头信息
self.head_name = config.get("header", "name")
self.head_data_len_len = int(config.get("header", "data"))
self.head_index_len = int(config.get("header", "index"))
self.head_time_len = int(config.get("header", "time"))
# 初始化队列大小信息
self.queue_size = int(config.get("receive", "queue_size"))
def init_connection(self):
try:
self.sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
self.sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
self.sock.bind(self.address)
except socket.error as msg:
print(msg)
sys.exit(1)
def init_connection_sock(self):
try:
sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
sock.bind(self.address)
return sock
except socket.error as msg:
print(msg)
sys.exit(1)
def close_connection(self):
self.sock.close()
def start(self):
# start threads to recieve
for i in range(self.packer.frame_pieces):
# intialize thread
thread = Thread(target=self.recv_thread, args=(i, ))
thread.daemon = True
thread.start()
decode_thread = Thread(target=self.rebuild_thread, args=(i, ))
decode_thread.daemon = True
decode_thread.start()
return self
def rebuild_thread(self, idx):
while True:
# 拥塞控制
if self.Q.qsize() > self.packer.piece_limit:
self.Q = Queue()
if self.Q.mutex:
self.Q.queue.clear()
# 不断地从队列里面取数据尝试
try:
pack = self.Q.get()
ptime = pack.ctime
loop = self.packer.frame_pieces - 1
# print(pack is not None)
while (pack is not None) and (loop >= 0):
idx = pack.idx
data = pack.frame
row_start = idx * self.packer.piece_size
row_end = (idx + 1) * self.packer.piece_size
self.frame[row_start:row_end] = data
if self.Q.qsize() == 0:
break
pack = self.Q.get()
loop -= 1
self.img_Q.put(
self.frame.reshape(self.packer.h, self.packer.w,
self.packer.d))
ctime = int(time.time() * 1000)
self.time_delay = ctime - ptime
except:
pass
return
def recv_thread(self, thread_idx):
sock = self.init_connection_sock()
stopped = False
while True:
if stopped: break
# otherwise, ensure the queue has room in it
if not self.Q.full():
try:
data, addr = sock.recvfrom(self.packer.pack_len)
idx, ctime, raw_img = self.packer.unpack_data(data)
line_data = numpy.frombuffer(raw_img, dtype=numpy.uint8)
line_data = cv2.imdecode(line_data, 1).flatten()
# add the frame to the queue
self.Q.put(FramePack(idx, ctime, line_data))
except:
pass
else:
time.sleep(0.01) # Rest for 10ms, we have a full queue
def read(self):
frame = self.Q.get()
return frame
if self.Q.qsize() > self.queue_size * 0.2: # self.queue_size*0.1
self.Q = Queue()
if self.Q.mutex:
self.Q.queue.clear()
return frame
# print(self.Q.qsize()) # 单机运行的时候，queue的长度持续增大
# 拥塞控制(这里可以用个算法,时间限制就写简单点)
now = int(time.time() * 1000)
if self.Q.qsize() == 0:
return None
while self.Q.qsize() > 0:
frame = self.Q.get()
ctime = frame.ctime
# select only when frametime is later than previous frame
if ctime >= self.last_frame_time:
# print("time-delay:",now - ctime," ms")
self.last_frame_time = ctime
break
if self.Q.qsize() > self.queue_size * 0.1: # self.queue_size*0.1
self.Q = Queue()
if self.Q.mutex:
self.Q.queue.clear()
return frame
def read_img(self):
# return self.imshow
# print(self.img_Q.qsize())
if self.img_Q.qsize() == 0:
return None
frame = self.img_Q.get()
if self.img_Q.qsize() > self.packer.frame_limit: # self.queue_size*0.1
# print("exeed limit")
self.img_Q = Queue()
if self.img_Q.mutex:
self.img_Q.queue.clear()
return frame
def read_show(self):
nvs = self.start()
last_frame_time = time.time()
tshow = 0
while True:
# print("reading img")
if cv2.waitKey(1) & 0xFF == ord('q'):
break
# now = time.time()
frame = self.read_img()
if frame is not None:
cnow = int(time.time() * 1000)
if cnow - nvs.delay_timer > 200:
nvs.delay_timer = cnow
tshow = nvs.time_delay
font = cv2.FONT_HERSHEY_SIMPLEX
bottomLeftCornerOfText = (10, 50)
fontScale = 1
fontColor = (0, 0, 255)
lineType = 2
cv2.putText(
frame,
'Get Fire! Recieve Delay: ' + str(tshow).ljust(3) + " ms",
bottomLeftCornerOfText, font, fontScale, fontColor,
lineType)
cv2.imshow("Receive server", frame)
last_frame_time = int(time.time() * 1000)
# print("piece show time is:", (now - last_frame_time))
def running(self):
return self.more() or not self.stopped
def more(self):
# return True if there are still frames in the queue. If stream is not stopped, try to wait a moment
tries = 0
while self.Q.qsize() == 0 and not self.stopped and tries < 5:
time.sleep(0.1)
tries += 1
return self.Q.qsize() > 0
def stop(self):
# indicate that the thread should be stopped
self.stopped = True
# wait until stream resources are released (producer thread might be still grabbing frame)
self.thread.join()
from packer import Packer
from fdates import parse
P = Packer()
P.d_csv(
1,
url="https://fred.stlouisfed.org/graph/fredgraph.csv?bgcolor=%23e1e9f0&chart_type=line&drp=0&fo=open%20sans&graph_bgcolor=%23ffffff&height=450&mode=fred&recession_bars=on&txtcolor=%23444444&ts=12&tts=12&width=1168&nt=0&thu=0&trc=0&show_legend=yes&show_axis_titles=yes&show_tooltip=yes&id=UNRATE&scale=left&cosd=1948-01-01&coed=2020-09-01&line_color=%234572a7&link_values=false&line_style=solid&mark_type=none&mw=3&lw=2&ost=-99999&oet=99999&mma=0&fml=a&fq=Monthly&fam=avg&fgst=lin&fgsnd=2020-02-01&line_index=1&transformation=lin&vintage_date=2020-10-29&revision_date=2020-10-29&nd=1948-01-01",
index="DATE",
subset="UNRATE",
name="number:Unemployment"
)
P.percentages(1, "UNRATE")
P.stacked(1, "UNRATE", inverse=True)
P.d_csv(
2,
url="https://fred.stlouisfed.org/graph/fredgraph.csv?bgcolor=%23e1e9f0&chart_type=line&drp=0&fo=open%20sans&graph_bgcolor=%23ffffff&height=450&mode=fred&recession_bars=on&txtcolor=%23444444&ts=12&tts=12&width=1168&nt=0&thu=0&trc=0&show_legend=yes&show_axis_titles=yes&show_tooltip=yes&id=INDPRO&scale=left&cosd=1919-01-01&coed=2020-09-01&line_color=%234572a7&link_values=false&line_style=solid&mark_type=none&mw=3&lw=2&ost=-99999&oet=99999&mma=0&fml=a&fq=Monthly&fam=avg&fgst=lin&fgsnd=2020-02-01&line_index=1&transformation=lin&vintage_date=2020-10-29&revision_date=2020-10-29&nd=1919-01-01",
index="DATE",
subset="INDPRO",
name="number:Industrial Production"
)
P.percentages(2, "INDPRO")
P.stacked(2, "INDPRO", inverse=True)
# P.d_csv(
# 3,
# url="https://fred.stlouisfed.org/graph/fredgraph.csv?bgcolor=%23e1e9f0&chart_type=line&drp=0&fo=open%20sans&graph_bgcolor=%23ffffff&height=450&mode=fred&recession_bars=on&txtcolor=%23444444&ts=12&tts=12&width=1168&nt=0&thu=0&trc=0&show_legend=yes&show_axis_titles=yes&show_tooltip=yes&id=BAMLH0A0HYM2&scale=left&cosd=2015-10-28&coed=2020-10-28&line_color=%234572a7&link_values=false&line_style=solid&mark_type=none&mw=3&lw=2&ost=-99999&oet=99999&mma=0&fml=a&fq=Daily%2C%20Close&fam=avg&fgst=lin&fgsnd=2020-02-01&line_index=1&transformation=lin&vintage_date=2020-10-29&revision_date=2020-10-29&nd=1996-12-31",
# index="DATE",
# subset="BAMLH0A0HYM2",
# name="number:Option Spreads"
def main():
args = parse_args()
packer = Packer(512, 512, padding=args.padding)
load(packer, args.directory)
packer.pack()
save(packer, 'test.png')
def update_from_export_data(self, kgk_manager, blob):
"""
Call this method to pull settings from the sync server.
:param kgk_manager: the kgk manager used for the decryption
:type kgk_manager: KgkManager
:param blob: the export data
:type blob: bytes
"""
if not blob[0] == 1:
print(
"Version error: Wrong data format. Could not import anything.")
return True
settings_crypter = self.get_settings_crypter(kgk_manager)
decrypted_settings = settings_crypter.decrypt(blob[145:])
if len(decrypted_settings) <= 0:
print("Wrong password.")
return False
self.remote_data = json.loads(
str(Packer.decompress(decrypted_settings), encoding='utf-8'))
self.update_remote = False
for domain_name in self.remote_data.keys():
data_set = self.remote_data[domain_name]
found = False
i = 0
while i < len(self.settings):
setting = self.settings[i]
if setting.get_domain() == domain_name:
found = True
if 'mDate' in data_set:
last_modification_date = data_set['mDate']
else:
last_modification_date = data_set['cDate']
if PasswordSetting.convert_ISO_date(
last_modification_date) > setting.get_m_date():
if 'deleted' in data_set and data_set['deleted']:
self.settings.pop(i)
else:
setting.load_from_dict(data_set)
setting.set_synced(True)
self.update_remote = True
i += 1
else:
i += 1
else:
i += 1
if not found:
new_setting = PasswordSetting(domain_name)
new_setting.load_from_dict(data_set)
new_setting.set_synced(True)
self.settings.append(new_setting)
for setting in self.settings:
found = False
for domain_name in self.remote_data.keys():
data_set = self.remote_data[domain_name]
if setting.get_domain() == domain_name:
found = True
if 'mDate' in data_set:
last_modification_date = data_set['mDate']
else:
last_modification_date = data_set['cDate']
if setting.get_m_date(
) >= PasswordSetting.convert_ISO_date(
last_modification_date):
self.update_remote = True
if not found:
self.update_remote = True
self.store_local_settings(kgk_manager)
return self.update_remote
def test_update_from_sync(self):
settings = {
'settings': {
'unit.test': {
'domain':
'unit.test',
'passwordTemplate':
'xxaAnoxxxxx',
'iterations':
5000,
'notes':
'Nice note!',
'salt':
'cGVwcGVy',
'usedCharacters':
'abcdefghijklmnopqrstuvwxyzABCDEFGHJKLMNPQRTUVWXYZ0123456789'
+ '#!"§$%&/()[]{}=-_+*<>;:.',
'cDate':
'2011-02-12T11:07:31',
'mDate':
'2011-02-12T11:07:32'
},
'some.domain': {
'domain': 'some.domain',
'passwordTemplate': 'oanA',
'iterations': 4096,
'salt': 'cGVwcGVy',
'usedCharacters': '6478593021',
'cDate': '2013-06-17T04:03:41',
'mDate': '2014-08-02T10:37:12'
}
},
'synced': []
}
salt = os.urandom(32)
kgk_manager = KgkManager()
kgk_manager.set_preference_manager(self.preference_manager)
kgk_manager.create_new_kgk()
kgk_block = kgk_manager.create_and_save_new_kgk_block(
Crypter(Crypter.createIvKey('xyz'.encode('utf-8'), salt)))
crypter = PasswordSettingsManager.get_settings_crypter(kgk_manager)
f = open(os.path.expanduser('~/.ctSESAM_test.pws'), 'bw')
f.write(salt + kgk_block + crypter.encrypt(
struct.pack('!I', 0) +
Packer.compress(json.dumps(settings).encode('utf-8'))))
f.close()
self.preference_manager.read_file()
self.manager.sync_manager = MockSyncManager(kgk_manager.get_kgk())
self.manager.load_settings(kgk_manager, 'xyz')
self.assertIn('unit.test', self.manager.get_domain_list())
self.assertIn('some.domain', self.manager.get_domain_list())
self.assertIn('third.domain', self.manager.get_domain_list())
self.assertEqual(
5001,
self.manager.get_setting('unit.test').get_iterations())
self.assertEqual(
4096,
self.manager.get_setting('some.domain').get_iterations())
self.assertEqual(
4098,
self.manager.get_setting('third.domain').get_iterations())
file = os.path.expanduser('~/.ctSESAM_test_extra.pws')
if os.path.isfile(file):
try:
import win32con
import win32api
win32api.SetFileAttributes(file,
win32con.FILE_ATTRIBUTE_NORMAL)
except ImportError:
pass
os.remove(file)
from packer import Packer
from fdates import parse
P = Packer()
P.d_fred(
1,
"https://alfred.stlouisfed.org/graph/alfredgraph.csv?bgcolor=%23e1e9f0&chart_type=column&drp=0&fo=open%20sans&graph_bgcolor=%23ffffff&height=450&mode=alfred&recession_bars=on&txtcolor=%23444444&ts=12&tts=12&width=1168&nt=0&thu=0&trc=0&show_legend=yes&show_axis_titles=yes&show_tooltip=yes&id=MRTSSM44511USN,MRTSSM44511USN&scale=left,left&cosd=2001-01-01,2001-01-01&coed=2020-07-01,2020-08-01&line_color=%234572a7,%23aa4643&link_values=false,false&line_style=solid,solid&mark_type=none,none&mw=3,3&lw=2,2&ost=-99999,-99999&oet=99999,99999&mma=0,0&fml=a,a&fq=Monthly,Monthly&fam=avg,avg&fgst=lin,lin&fgsnd=2020-02-01,2020-02-01&line_index=1,2&transformation=lin,lin&vintage_date=2020-09-16,2020-10-16&nd=2001-01-01,2001-01-01",
"MRTSSM44511USN_20201016")
# P.percentages(1, "MRTSSM44511USN_20201016")
# P.stacked(1, "MRTSSM44511USN_20201016")
# P.d_yfi(2, "^VIX")
# P.percentages(2, "close")
# P.inverse(2, "close")
# P.add(2, "close", 0.02)
# P.stacked(2, "close")
P.d_yfi(2, "wow.ax")
# P.percentages(2, "close")
# P.stacked(2, "close")
# P.d_yfi(3, "col.ax")
# P.percentages(3, "close")
# P.stacked(3, "close")
# P.dataset(2, {
# "url": "^VIX.yfi",
# "range": "daily",
# "name": "VIX",
# "index": "date",