def test_config_generation_ok(self):
'''This test case ensures config nginx script works as expected.'''
settings_cls = SettingsFacade
root_folder = os.path.abspath(instantiator.get_class_abslocation(settings_cls) + "../")
tpl_loader = FileSystemLoader(searchpath=root_folder)
tpl_env = Environment(loader=tpl_loader)
config_data = {"ip_address": "127.0.0.1",
"vhost_name": "test-app.com",
"http_port": 80,
"uwsgi_port": 12090,
"root_folder": "/test/folder/vhost",
"modules_folder": "/"}
expected_config = tpl_env.get_template("/deployment/conf/nginx/fantastico-wsgi").render(config_data)
sys.argv = ["config_nginx.py",
"--ipaddress", "127.0.0.1",
"--vhost-name", "test-app.com",
"--uwsgi-port", "12090",
"--root-folder", "/test/folder/vhost",
"--modules-folder", "/"]
from fantastico.deployment import config_nginx
generated_config = StringIO()
config_nginx.main(generated_config)
self.assertEqual(expected_config, generated_config.getvalue())
def _open(self):
class Writer(XMLGenerator):
__qualname__ = 'TuningDescFileWriter._open.<locals>.Writer'
SPACES_PER_INDENT = 4
def __init__(self, *args, whitespace_depth=0, **kwargs):
super().__init__(*args, **kwargs)
self._whitespace_depth = whitespace_depth
self._indent = 0
self._already_closed = None
self._last_indent = -1
def startElement(self, name, attrs, can_close=False):
sub_elements = {}
if self._indent == self._last_indent and self._indent <= self.SPACES_PER_INDENT * self._whitespace_depth:
self.add_new_line()
self._last_indent = self._indent
self._write('{0}<{1}'.format(' ' * self._indent, name))
for (attr_name, value) in attrs.items():
if isinstance(value, dict) or isinstance(value, list):
sub_elements[attr_name] = value
else:
while value is not None:
self._write(' %s=%s' %
(attr_name, quoteattr(str(value))))
if not sub_elements and can_close:
self._write(' />\n'.format(name))
self._already_closed = name
else:
self._write('>\n')
for (name, value) in sub_elements.items():
if isinstance(value, dict):
self.startElement(name, value, can_close=True)
self.endElement(name)
else:
for sub_item in value:
self.startElement(name,
sub_item,
can_close=True)
self.endElement(name)
def endElement(self, name):
if self._already_closed is not None:
self._already_closed = None
else:
self._write(' ' * self._indent)
super().endElement(name)
self.add_new_line()
self._last_indent = self._indent
def add_new_line(self):
self._write('\n')
self._string = StringIO()
self._writer = Writer(self._string,
whitespace_depth=self._whitespace_depth)
self._writer.startDocument()
self._writer.add_new_line()
def test_get_info_with_params(self):
js = {"row": 1, "in": 2, "out": 3}
jsret = {"num_row": 1, "num_in": 2, "num_out": 3}
filelike = StringIO(json.dumps(js))
with patch.object(self.api, "_fetch_data", return_value=[filelike, None]) as api:
f = self.api.get_info()
api.assert_called_with("datainfo", params={})
self.assertEquals(jsret, f)
filelike.seek(0)
with patch.object(self.api, "_fetch_data", return_value=[filelike, None]) as api:
f = self.api.get_info(multiply=3)
api.assert_called_with("datainfo", params={"multiply": 3})
self.assertEquals(jsret, f)
def test_random_read(self):
'''
Test a reading from a random sample, with a random size and a random
start address
'''
sample = random.sample(self.samples, 1)[0]
cfg.filename = sample
self.assertTrue(os.path.exists(sample))
sample_size = os.path.getsize(sample)
start = random.randrange(sample_size)
size_range = sample_size - start
dump_size = random.randrange(size_range)
end = start + dump_size
sys.stdout = StringIO() # Suppress output
memdump.dump(start, end)
sys.stdout = sys.__stdout__ # Restore output
output_fn = '{0}_{1}-{2}.bin'.format(cfg.memdump_prefix, hex(start),
hex(end))
self.assertTrue(os.path.exists(output_fn))
md5 = hashlib.md5()
f = open(sample, 'rb')
f.seek(start)
read = f.read(dump_size)
md5.update(read)
self.assertEqual(md5.digest(), self.file_md5(output_fn))
f.close()
def test_random_read(self):
'''
Test a reading from a random sample, with a random size and a random
start address
'''
sample = random.sample(self.samples, 1)[0]
self.opts.filename = sample
self.assertTrue(os.path.exists(sample))
sample_size = os.path.getsize(sample)
self.opts.address = random.randrange(sample_size)
size_range = sample_size - self.opts.address
self.opts.size = random.randrange(size_range)
sys.stdout = StringIO() # Suppress output
device, memsize = interface.initialize(self.opts, self.module)
memspace = memory.MemorySpace(device, memsize)
dump.run(self.opts, memspace)
sys.stdout = sys.__stdout__ # Restore output
output_fn = dump.filename
self.assertTrue(os.path.exists(output_fn))
md5 = hashlib.md5()
f = open(sample, 'rb')
f.seek(self.opts.address)
read = f.read(self.opts.size)
md5.update(read)
self.assertEqual(md5.digest(), self.file_md5(output_fn))
f.close()
def test_unlock(self):
for sample in self.samples:
cfg.startaddress = 0x00000000
mod_name = sample[0]
filepath = sample[1]
try:
module = SourceFileLoader(mod_name, filepath).load_module()
except ImportError:
assert (module)
self.opts.interface = 'file'
self.opts.filename = path.join(path.dirname(__file__),
'samples/') + mod_name + '.bin'
foundtarget = False
for i, target in enumerate(unlock.targets, 1):
if target.signatures[0].os == module.OS:
foundtarget = [target]
self.opts.target_number = i
# print(module.OS)
self.assertTrue(foundtarget)
self.assertIsNotNone(self.opts.target_number)
module.IS_INTRUSIVE = True
sys.stdout = StringIO() # Suppress output
device, memsize = interface.initialize(self.opts, module)
memspace = memory.MemorySpace(device, memsize)
address, page = unlock.run(self.opts, memspace)
sys.stdout = sys.__stdout__ # Restore output
# print(address & 0x00000fff)
# print(module.offset)
#self.assertEqual(address & 0x00000fff, module.offset)
self.assertEqual(page, module.page)
def test_prnt(self):
s = ''.join(map(str, range(20)))
cfg.termwidth = 10
sys.stdout = StringIO() # Suppress output
cfg.wrapper.width = util.get_termsize()
sys.stdout = StringIO() # Suppress output
sys.stdout.write('')
print_wrapped(s)
out = sys.stdout.getvalue()
sys.stdout = sys.__stdout__ # Restore output
self.assertEqual(out,
'0123456789\n' +
' 101112\n' +
' 131415\n' +
' 161718\n' +
' 19\n')
def test_screenlock(self):
for sample in self.samples:
settings = imp.reload(inception.settings)
settings.startaddress = 0x00000000
mod_name = sample[0]
filepath = sample[1]
#util.msg('T', 'Testing sample {0}'.format(mod_name))
try:
module = imp.load_source(mod_name, filepath)
except ImportError:
assert (module)
settings.filemode = True
settings.filename = path.join(path.dirname(__file__),
'../samples/') + mod_name + '.bin'
foundtarget = False
for target in settings.targets:
if target['OS'] == module.OS:
foundtarget = [target]
self.assertTrue(foundtarget)
#util.msg('T', 'Found target: {0}'.format(foundtarget[0]['OS']))
sys.stdout = StringIO() # Supress output
address, page = screenlock.attack(foundtarget)
sys.stdout = sys.__stdout__ # Restore output
self.assertEqual(address & 0x00000fff, module.offset)
self.assertEqual(page, module.page)
def test_findtag(self):
target = targets[2]
sys.stdout = StringIO() # Suppress output
device, memsize = interface.initialize(self.opts, self.module)
memspace = memory.MemorySpace(device, memsize)
results = memspace.find(target, findtag=True)
sys.stdout = sys.__stdout__ # Restore output
self.assertEqual(len(results), 2)
self.assertEqual(results[1][1], targets[2].signatures[1])
def test_find(self):
target = targets[0]
sys.stdout = StringIO() # Suppress output
device, memsize = interface.initialize(self.opts, self.module)
memspace = memory.MemorySpace(device, memsize)
address, signature, offset = memspace.find(target)
sys.stdout = sys.__stdout__ # Restore output
self.assertEqual(address, 672)
self.assertEqual(signature, target.signatures[0])
self.assertEqual(offset, target.signatures[0].offsets[0])
def _open(self):
class Writer(XMLGenerator):
__qualname__ = 'TuningDescFileWriter._open.<locals>.Writer'
SPACES_PER_INDENT = 4
def __init__(self, *args, whitespace_depth=0, **kwargs):
super().__init__(*args, **kwargs)
self._whitespace_depth = whitespace_depth
self._indent = 0
self._already_closed = None
self._last_indent = -1
def startElement(self, name, attrs, can_close=False):
sub_elements = {}
if self._indent == self._last_indent and self._indent <= self.SPACES_PER_INDENT*self._whitespace_depth:
self.add_new_line()
self._last_indent = self._indent
self._write('{0}<{1}'.format(' '*self._indent, name))
for (attr_name, value) in attrs.items():
if isinstance(value, dict) or isinstance(value, list):
sub_elements[attr_name] = value
else:
while value is not None:
self._write(' %s=%s' % (attr_name, quoteattr(str(value))))
if not sub_elements and can_close:
self._write(' />\n'.format(name))
self._already_closed = name
else:
self._write('>\n')
for (name, value) in sub_elements.items():
if isinstance(value, dict):
self.startElement(name, value, can_close=True)
self.endElement(name)
else:
for sub_item in value:
self.startElement(name, sub_item, can_close=True)
self.endElement(name)
def endElement(self, name):
if self._already_closed is not None:
self._already_closed = None
else:
self._write(' '*self._indent)
super().endElement(name)
self.add_new_line()
self._last_indent = self._indent
def add_new_line(self):
self._write('\n')
self._string = StringIO()
self._writer = Writer(self._string, whitespace_depth=self._whitespace_depth)
self._writer.startDocument()
self._writer.add_new_line()
def test_fulldump(self):
start = 0x00000000
for sample in self.samples:
cfg.filename = sample
end = os.path.getsize(sample)
sys.stdout = StringIO() # Suppress output
memdump.dump(start, end)
sys.stdout = sys.__stdout__ # Restore output
output_fn = '{0}_{1}-{2}.bin'.format(cfg.memdump_prefix,hex(start), hex(end))
self.assertTrue(os.path.exists(output_fn))
self.assertEqual(self.file_md5(sample), self.file_md5(output_fn))
def test_fulldump(self):
start = 0x00000000
for sample in self.samples:
cfg.filename = sample
end = os.path.getsize(sample)
sys.stdout = StringIO() # Suppress output
memdump.dump(start, end)
sys.stdout = sys.__stdout__ # Restore output
output_fn = memdump.filename
self.assertTrue(os.path.exists(output_fn))
self.assertEqual(self.file_md5(sample), self.file_md5(output_fn))
def test_findall(self):
target = targets[1]
sys.stdout = StringIO() # Suppress output
device, memsize = interface.initialize(self.opts, self.module)
memspace = memory.MemorySpace(device, memsize)
results = memspace.find(target, findall=True)
sys.stdout = sys.__stdout__ # Restore output
self.assertEqual(len(results), 4)
self.assertEqual(results[0][0], 49)
self.assertEqual(results[3][0], 16433)
for result in results:
self.assertEqual(result[1], target.signatures[0])
def test_fulldump(self):
for sample in self.samples:
self.opts.interface = 'file'
self.opts.filename = sample
sys.stdout = StringIO() # Suppress output
device, memsize = interface.initialize(self.opts, self.module)
memspace = memory.MemorySpace(device, memsize)
dump.run(self.opts, memspace)
sys.stdout = sys.__stdout__ # Restore output
output_fn = dump.filename
self.assertTrue(os.path.exists(output_fn))
self.assertEqual(self.file_md5(sample), self.file_md5(output_fn))
def test_write(self):
s = 'A' * (3 * term.size())
cfg.wrapper.width = term.size()
sys.stdout = StringIO() # Suppress output
sys.stdout.write('')
term.write(s)
out = sys.stdout.getvalue()
sys.stdout = sys.__stdout__ # Restore output
expected = 'A' * term.size()
n = term.size()
expected = 'A' * term.size() + '\n '
t = 'A' * (2 * term.size())
expected = expected + '\n '.join([t[i:i+n-4] for i in range(0, len(t) -4 , n-4)]) + '\n'
self.assertEqual(out, expected)
def _yield_lines(self, tokensList):
"""
we iterate over all datasources as long as one is available hence
still contains tokens
"""
lineno = 1
decoratedTokensList = []
for index, tokenSource in enumerate(tokensList):
decoratedTokens = {
"notEmpty": True,
"tokensource": tokenSource,
"index": index,
# this generator replaces temporally the generator returned by the lexer
# in order to support lexers returning tokens that contain line feeds
# within
"yieldTokenParts": None,
}
decoratedTokensList.append(decoratedTokens)
while self._is_code_left(decoratedTokensList):
lineBuffer = StringIO()
lineStarted = False
lineEmpty = True
for containsSomething, singleSourceLine in self._yield_multisourceline(decoratedTokensList, lineno):
if containsSomething:
lineEmpty = False
if not lineStarted:
lineBuffer.write("<tr>")
for linenoPart in self._yield_lineno(lineno):
lineBuffer.write(linenoPart)
lineno += 1
lineStarted = True
lineBuffer.write(singleSourceLine)
if lineStarted:
lineBuffer.write("</tr>")
yield lineEmpty, lineBuffer.getvalue()
def test_patch(self):
target = targets[3]
sig = target.signatures[0]
sys.stdout = StringIO() # Suppress output
device, memsize = interface.initialize(self.opts, self.module)
device.dry_run = False
memspace = memory.MemorySpace(device, memsize)
address = 0x00000042
read = memspace.read(address, 4)
backup = memspace.patch(address, sig.chunks)
sys.stdout = sys.__stdout__ # Restore output
read_back = memspace.read(address, 4)
# print(read_back)
self.assertEqual(sig.chunks[0].patch, read_back)
memspace.write(address, read)
read_back = memspace.read(address, 4)
# print(read_back)
self.assertEqual(read, read_back)
def test_screenlock(self):
for sample in self.samples:
cfg = imp.reload(inception.cfg)
cfg.startaddress = 0x00000000
mod_name = sample[0]
filepath = sample[1]
try:
module = imp.load_source(mod_name, filepath)
except ImportError:
assert (module)
cfg.filemode = True
cfg.filename = path.join(path.dirname(__file__),
'samples/') + mod_name + '.bin'
foundtarget = False
for target in cfg.targets:
if target['OS'] == module.OS:
foundtarget = [target]
self.assertTrue(foundtarget)
sys.stdout = StringIO() # Suppress output
address, page = screenlock.attack(foundtarget)
sys.stdout = sys.__stdout__ # Restore output
self.assertEqual(address & 0x00000fff, module.offset)
self.assertEqual(page, module.page)
def __enter__(self):
if self.disable:
sys.stdout = StringIO()
class TuningDescFileWriter:
__qualname__ = 'TuningDescFileWriter'
SORT_OVERRIDE = collections.defaultdict(lambda : 0, {'name': 100, 'class': 99, 'type': 98, 'default': 97, 'min': -1, 'max': -2, 'description': -100})
def __init__(self, module, export_path, whitespace_depth=0):
if export_path is None:
export_path = paths.TUNING_ROOTS[sims4.resources.Types.TUNING]
tuning_file = self.get_file_name(module)
self._filename = os.path.join(export_path, tuning_file)
self._writer = None
self._whitespace_depth = whitespace_depth
def get_file_name(self, module):
is_fragment = getattr(module, 'is_fragment', False)
if is_fragment:
return get_tdesc_frag_name(module)
return get_desc_file_name(module)
def open(self):
self._open()
@staticmethod
def list_key(value):
if isinstance(value, dict):
return value.get(Attributes.Name, '')
return ''
@staticmethod
def sort_tags_recursive(attr_vals, sort_override=None):
if not sort_override:
sort_key = None
else:
def sort_key(value):
return (-sort_override[value], value)
if isinstance(attr_vals, dict) and not isinstance(attr_vals, collections.OrderedDict):
new_vals = collections.OrderedDict()
for key in sorted(attr_vals, key=sort_key):
new_vals[key] = TuningDescFileWriter.sort_tags_recursive(attr_vals[key], sort_override)
return new_vals
if isinstance(attr_vals, list):
new_vals = []
for value in sorted(attr_vals, key=TuningDescFileWriter.list_key):
new_vals.append(TuningDescFileWriter.sort_tags_recursive(value, sort_override))
return new_vals
return attr_vals
def write_tunable(self, tunable):
desc_tag = tunable.TAGNAME
attr_vals = tunable.export_desc()
sorted_vals = TuningDescFileWriter.sort_tags_recursive(attr_vals, sort_override=TuningDescFileWriter.SORT_OVERRIDE)
self._writer.startElement(desc_tag, AttributesImpl(sorted_vals), can_close=True)
self._writer.endElement(desc_tag)
def write_frag(self, tunable):
desc_tag = Attributes.TdescFragClass
tunable_vals = {}
attr_vals = tunable.frag_desc()
tunable_vals[tunable.FRAG_TAG_NAME] = attr_vals
sorted_vals = TuningDescFileWriter.sort_tags_recursive(tunable_vals, sort_override=TuningDescFileWriter.SORT_OVERRIDE)
self._writer.startElement(desc_tag, AttributesImpl(sorted_vals), can_close=True)
self._writer.endElement(desc_tag)
def write_enum_items(self, enum_class):
if hasattr(enum_class, '_static_index'):
last_static_index = enum_class._static_index + 1
else:
last_static_index = len(enum_class.names)
for i in range(last_static_index):
enum_name = enum_class.names[i]
enum_value = int(enum_class[enum_name])
attr_vals = {Attributes.Name: enum_name, Attributes.EnumValue: enum_value}
self._writer.startElement(Tags.EnumItem, AttributesImpl(attr_vals), can_close=True)
self._writer.endElement(Tags.EnumItem)
def write_deleted_tunable(self, deleted_tunable_name):
attr_vals = {Attributes.Name: deleted_tunable_name}
self._writer.startElement(Tags.Deleted, AttributesImpl(attr_vals), can_close=True)
self._writer.endElement(Tags.Deleted)
def _open(self):
class Writer(XMLGenerator):
__qualname__ = 'TuningDescFileWriter._open.<locals>.Writer'
SPACES_PER_INDENT = 4
def __init__(self, *args, whitespace_depth=0, **kwargs):
super().__init__(*args, **kwargs)
self._whitespace_depth = whitespace_depth
self._indent = 0
self._already_closed = None
self._last_indent = -1
def startElement(self, name, attrs, can_close=False):
sub_elements = {}
if self._indent == self._last_indent and self._indent <= self.SPACES_PER_INDENT*self._whitespace_depth:
self.add_new_line()
self._last_indent = self._indent
self._write('{0}<{1}'.format(' '*self._indent, name))
for (attr_name, value) in attrs.items():
if isinstance(value, dict) or isinstance(value, list):
sub_elements[attr_name] = value
else:
while value is not None:
self._write(' %s=%s' % (attr_name, quoteattr(str(value))))
if not sub_elements and can_close:
self._write(' />\n'.format(name))
self._already_closed = name
else:
self._write('>\n')
for (name, value) in sub_elements.items():
if isinstance(value, dict):
self.startElement(name, value, can_close=True)
self.endElement(name)
else:
for sub_item in value:
self.startElement(name, sub_item, can_close=True)
self.endElement(name)
def endElement(self, name):
if self._already_closed is not None:
self._already_closed = None
else:
self._write(' '*self._indent)
super().endElement(name)
self.add_new_line()
self._last_indent = self._indent
def add_new_line(self):
self._write('\n')
self._string = StringIO()
self._writer = Writer(self._string, whitespace_depth=self._whitespace_depth)
self._writer.startDocument()
self._writer.add_new_line()
def start_namespace(self, namespace, contents):
attribute_dict = {}
for (attribute, value) in contents.items():
while not isinstance(value, TunableBase) and (not isinstance(value, dict) and (not isinstance(value, list) and not isinstance(value, enum.Metaclass))) and value is not DELETEDMARKER:
attribute_dict[attribute] = value
self._writer.startElement(namespace, attribute_dict)
def end_namespace(self, namespace):
self._writer.endElement(namespace)
def close(self):
self._writer.endDocument()
serialized_data = self._string.getvalue()
self._string.close()
path = os.path.split(self._filename)[0]
if path and not os.path.exists(path):
os.makedirs(path)
do_compare = True
tuning_file = None
created = True
if os.path.exists(self._filename):
created = False
if not do_compare:
try:
with open(self._filename, 'w'):
pass
except IOError:
do_compare = True
if do_compare:
with open(self._filename, 'r') as old_tuning_file:
old_serialized_data = old_tuning_file.read()
if serialized_data == old_serialized_data:
logger.debug('Skipped tuning file: {}', self._filename)
return
try:
tuning_file = open(self._filename, 'w')
except IOError:
raise TunableFileReadOnlyError(self._filename)
with tuning_file:
tuning_file.write(serialized_data)
if created:
logger.warn('CREATED tuning file: {}', self._filename)
elif do_compare:
logger.info('Updated tuning file: {}', self._filename)
else:
logger.info('Wrote tuning file: {}', self._filename)
# go to captcha page
captcha_page_response = HttpUtility.sendHttpRequest(
baseUrl + javascript_var_parser.get_captchaUrl(), {'Referer': url},
None, cookies)
captcha_html_page = HttpUtility.getHtmlPage(captcha_page_response)
#print(captcha_html_page) write to file instead for debug
if captcha_html_page.strip() != "":
OtherUtility.write_file("captcha_page.html",
captcha_html_page.encode('utf-8'))
else:
print("Empty response.")
'''pahse 2'''
'''get download link'''
download_url = ParserUtility.find_download_link(
StringIO(captcha_html_page))
if (download_url == None):
print(captcha_page_response.read())
print("Download link not found")
sys.exit()
else:
print("Going to download.")
print(download_url)
captcha_page_response = HttpUtility.sendHttpRequest(
download_url, {
'Referer': url,
'Connection': "keep-alive"
}, None, cookies)
OtherUtility.write_file("aaa.zip", captcha_page_response.read())
'''phase 2'''
def __enter__(self):
if self.disable:
self.original_stdout = sys.stdout
sys.stdout = StringIO()
class TuningDescFileWriter:
__qualname__ = 'TuningDescFileWriter'
SORT_OVERRIDE = collections.defaultdict(
lambda: 0, {
'name': 100,
'class': 99,
'type': 98,
'default': 97,
'min': -1,
'max': -2,
'description': -100
})
def __init__(self, module, export_path, whitespace_depth=0):
if export_path is None:
export_path = paths.TUNING_ROOTS[sims4.resources.Types.TUNING]
tuning_file = self.get_file_name(module)
self._filename = os.path.join(export_path, tuning_file)
self._writer = None
self._whitespace_depth = whitespace_depth
def get_file_name(self, module):
is_fragment = getattr(module, 'is_fragment', False)
if is_fragment:
return get_tdesc_frag_name(module)
return get_desc_file_name(module)
def open(self):
self._open()
@staticmethod
def list_key(value):
if isinstance(value, dict):
return value.get(Attributes.Name, '')
return ''
@staticmethod
def sort_tags_recursive(attr_vals, sort_override=None):
if not sort_override:
sort_key = None
else:
def sort_key(value):
return (-sort_override[value], value)
if isinstance(attr_vals, dict) and not isinstance(
attr_vals, collections.OrderedDict):
new_vals = collections.OrderedDict()
for key in sorted(attr_vals, key=sort_key):
new_vals[key] = TuningDescFileWriter.sort_tags_recursive(
attr_vals[key], sort_override)
return new_vals
if isinstance(attr_vals, list):
new_vals = []
for value in sorted(attr_vals, key=TuningDescFileWriter.list_key):
new_vals.append(
TuningDescFileWriter.sort_tags_recursive(
value, sort_override))
return new_vals
return attr_vals
def write_tunable(self, tunable):
desc_tag = tunable.TAGNAME
attr_vals = tunable.export_desc()
sorted_vals = TuningDescFileWriter.sort_tags_recursive(
attr_vals, sort_override=TuningDescFileWriter.SORT_OVERRIDE)
self._writer.startElement(desc_tag,
AttributesImpl(sorted_vals),
can_close=True)
self._writer.endElement(desc_tag)
def write_frag(self, tunable):
desc_tag = Attributes.TdescFragClass
tunable_vals = {}
attr_vals = tunable.frag_desc()
tunable_vals[tunable.FRAG_TAG_NAME] = attr_vals
sorted_vals = TuningDescFileWriter.sort_tags_recursive(
tunable_vals, sort_override=TuningDescFileWriter.SORT_OVERRIDE)
self._writer.startElement(desc_tag,
AttributesImpl(sorted_vals),
can_close=True)
self._writer.endElement(desc_tag)
def write_enum_items(self, enum_class):
if hasattr(enum_class, '_static_index'):
last_static_index = enum_class._static_index + 1
else:
last_static_index = len(enum_class.names)
for i in range(last_static_index):
enum_name = enum_class.names[i]
enum_value = int(enum_class[enum_name])
attr_vals = {
Attributes.Name: enum_name,
Attributes.EnumValue: enum_value
}
self._writer.startElement(Tags.EnumItem,
AttributesImpl(attr_vals),
can_close=True)
self._writer.endElement(Tags.EnumItem)
def write_deleted_tunable(self, deleted_tunable_name):
attr_vals = {Attributes.Name: deleted_tunable_name}
self._writer.startElement(Tags.Deleted,
AttributesImpl(attr_vals),
can_close=True)
self._writer.endElement(Tags.Deleted)
def _open(self):
class Writer(XMLGenerator):
__qualname__ = 'TuningDescFileWriter._open.<locals>.Writer'
SPACES_PER_INDENT = 4
def __init__(self, *args, whitespace_depth=0, **kwargs):
super().__init__(*args, **kwargs)
self._whitespace_depth = whitespace_depth
self._indent = 0
self._already_closed = None
self._last_indent = -1
def startElement(self, name, attrs, can_close=False):
sub_elements = {}
if self._indent == self._last_indent and self._indent <= self.SPACES_PER_INDENT * self._whitespace_depth:
self.add_new_line()
self._last_indent = self._indent
self._write('{0}<{1}'.format(' ' * self._indent, name))
for (attr_name, value) in attrs.items():
if isinstance(value, dict) or isinstance(value, list):
sub_elements[attr_name] = value
else:
while value is not None:
self._write(' %s=%s' %
(attr_name, quoteattr(str(value))))
if not sub_elements and can_close:
self._write(' />\n'.format(name))
self._already_closed = name
else:
self._write('>\n')
for (name, value) in sub_elements.items():
if isinstance(value, dict):
self.startElement(name, value, can_close=True)
self.endElement(name)
else:
for sub_item in value:
self.startElement(name,
sub_item,
can_close=True)
self.endElement(name)
def endElement(self, name):
if self._already_closed is not None:
self._already_closed = None
else:
self._write(' ' * self._indent)
super().endElement(name)
self.add_new_line()
self._last_indent = self._indent
def add_new_line(self):
self._write('\n')
self._string = StringIO()
self._writer = Writer(self._string,
whitespace_depth=self._whitespace_depth)
self._writer.startDocument()
self._writer.add_new_line()
def start_namespace(self, namespace, contents):
attribute_dict = {}
for (attribute, value) in contents.items():
while not isinstance(value, TunableBase) and (
not isinstance(value, dict) and
(not isinstance(value, list) and not isinstance(
value, enum.Metaclass))) and value is not DELETEDMARKER:
attribute_dict[attribute] = value
self._writer.startElement(namespace, attribute_dict)
def end_namespace(self, namespace):
self._writer.endElement(namespace)
def close(self):
self._writer.endDocument()
serialized_data = self._string.getvalue()
self._string.close()
path = os.path.split(self._filename)[0]
if path and not os.path.exists(path):
os.makedirs(path)
do_compare = True
tuning_file = None
created = True
if os.path.exists(self._filename):
created = False
if not do_compare:
try:
with open(self._filename, 'w'):
pass
except IOError:
do_compare = True
if do_compare:
with open(self._filename, 'r') as old_tuning_file:
old_serialized_data = old_tuning_file.read()
if serialized_data == old_serialized_data:
logger.debug('Skipped tuning file: {}', self._filename)
return
try:
tuning_file = open(self._filename, 'w')
except IOError:
raise TunableFileReadOnlyError(self._filename)
with tuning_file:
tuning_file.write(serialized_data)
if created:
logger.warn('CREATED tuning file: {}', self._filename)
elif do_compare:
logger.info('Updated tuning file: {}', self._filename)
else:
logger.info('Wrote tuning file: {}', self._filename)
def run(self):
'''if self.op == 0:
self.setStatusInfo("Ready!!")
self.goToNextOP()'''
if self.op == 1:
''' connect to file page and check it is allowed to download right now'''
print("111111")
self.setStatusInfo("Connect to file url...")
http_page_response = HttpUtility.sendHttpRequest(
self.fileUrl, None, None, self.cookies)
start_html_page = HttpUtility.getHtmlPage(http_page_response)
# parse the java script var in downloading page
self.javascript_var_parser = ParserUtility.HTMLTagParser(
True, self.fileUrl)
self.javascript_var_parser.feed(start_html_page)
if self.javascript_var_parser.get_downloadInfo().strip() != "":
self.setStatusInfo(
"Error! " + self.javascript_var_parser.get_downloadInfo())
print(self.javascript_var_parser.get_downloadInfo())
self.setCurrentWorkDone()
print("111111done")
return
if self.op == 2:
''' press slow download btn (#btn-free)'''
print("2222")
self.setStatusInfo("Starting download procedure.")
fid_param = urllib.parse.urlencode(
{'fid': self.javascript_var_parser.get_fid()})
sid_response = HttpUtility.sendAjaxRequest(
self.baseUrl +
self.javascript_var_parser.get_startTimerUrl().strip() + "?" +
fid_param, {'Referer': self.fileUrl}, None, self.cookies)
sid_result = json.loads(sid_response.read().decode('utf-8'))
if sid_result['state'] != "started":
self.setStatusInfo("Error! after press slow download button.")
print("Error! after press slow download button.")
print("Stop!!!")
self.setCurrentWorkDone()
return
self.javascript_var_parser.set_sid(sid_result['sid'])
self.setCurrentWorkDone()
print("2222done")
return
if self.op == 3:
''' start counting down ( at startTimer() )'''
print("33333")
for left_time in range(
int(self.javascript_var_parser.get_secs()) + 1, 0, -1):
if self.needStop() == True:
print("User terminate!")
return
self.setStatusInfo("Remaining: " + str(left_time) + " sec.")
print("Remaining: ", left_time)
time.sleep(1)
self.setCurrentWorkDone()
print("33333done")
return
if self.op == 4:
''' get download link ( at getDownloadLink() )'''
print("44444")
self.setStatusInfo("Starting download procedure continue...")
sid_param = urllib.parse.urlencode(
{'sid': self.javascript_var_parser.get_sid()})
get_download_link_response = HttpUtility.sendAjaxRequest(
self.baseUrl +
self.javascript_var_parser.get_getDownloadUrl().strip() + "?" +
sid_param, {'Referer': self.fileUrl}, None, self.cookies)
self.get_download_link_result = json.loads(
get_download_link_response.read().decode('utf-8'))
if self.get_download_link_result['state'] != "done":
self.setStatusInfo("Error! after get download link.")
print("Error! after get download link.")
print("Stop!!!")
self.setCurrentWorkDone()
return
self.setCurrentWorkDone()
print("44444done")
return
if self.op == 5:
''' go to download page '''
print("55555")
self.setStatusInfo("Redirect to download url...")
captcha_page_response = HttpUtility.sendHttpRequest(
self.baseUrl + self.javascript_var_parser.get_captchaUrl(),
{'Referer': self.fileUrl}, None, self.cookies)
self.captcha_html_page = HttpUtility.getHtmlPage(
captcha_page_response)
self.download_url = ParserUtility.find_download_link(
StringIO(self.captcha_html_page))
if (self.download_url == None):
self.setStatusInfo("Error! Download link not found")
print("Error! Download link not found")
self.setCurrentWorkDone()
return
self.setCurrentWorkDone()
print("55555done")
return
if self.op == 6:
''' start to download '''
print("66666")
self.setStatusInfo("Prepare to download...")
if self.download_url == None:
self.setStatusInfo("Error! Download link not found")
print("Error! Download link not found")
self.setCurrentWorkDone()
return
# Not really download on testing
#captcha_page_response=HttpUtility.sendHttpRequest(self.download_url, {'Referer': self.fileUrl, 'Connection': "keep-alive"}, None, self.cookies)
#OtherUtility.write_file("aaa.zip", captcha_page_response.read())
self.setStatusInfo("Start to downloading...")
download_response = HttpUtility.sendHttpRequest(
self.download_url, {
'Referer': self.fileUrl,
'Connection': "keep-alive"
}, None, self.cookies)
write_result = OtherUtility.write_download_file(
download_response, self.FILE_NAME, self.CHUNK_SIZE,
self.setStatusInfo, self.needStop)
if write_result == 1:
self.setCurrentWorkDone()
self.setStatusInfo("DownLoad Fail!!! the downloaded is " +
self.FILE_NAME + ".tmp")
return
self.setCurrentWorkDone()
self.setStatusInfo("DownLoad success!!! save to file " +
self.FILE_NAME)
print("66666done")
return
def reload_module_from_string(module, source):
stream = StringIO(source)
filename = module.__dict__['__file__']
kind = imp.PY_SOURCE
return _reload(module, filename, stream, kind)