def _git_exec(self, *args, decode=True):
"""Done 'git' command."""
master_program = 'git'
command_tup = (master_program, *args)
logger.debug('runing command: %s', ' '.join(command_tup))
output = subprocess.check_output(command_tup, cwd=self.target_project_path,
stderr=subprocess.STDOUT)
return output.decode() if decode else output
def forward_diff(self):
"""Comparison between versions one by one from new to old."""
version_lst = self.version_lst
self._git_exec('checkout', self.version_lst[0].vstring)
logger.verbose('forward version list %s', version_lst)
self.last_hash(version_lst, 'fingerprint')
self.make_diff(version_lst, 'fingerprint')
self._git_exec('checkout', '-')
def show_all():
"""show all plugin, --all/-a option."""
logger.info('show all plugin introduction')
table_lst = [('framework name', 'tags')]
for plugin_info in all_plugin():
tags = ', '.join(plugin_info.get('alias'))
table_lst.append((plugin_info.get('framework'), tags))
table = AsciiTable(table_lst)
show_output(table.table)
def main():
filter_word = ''
try:
filter_word = sys.argv[1]
except IndexError:
pass
current_module = sys.modules[__name__]
for member, module in inspect.getmembers(current_module):
if member.endswith('_test') and filter_word in member:
logger.noise("test function: %s, module: %s", member, str(module))
getattr(current_module, member)()
def last_hash(self, version_lst, keyname):
"""Get all static file's hash string."""
static_file_lst = self.last_static()
last_ver = str(version_lst[0])
logger.verbose('last hash dictionary, key: %s, version: %s', keyname, last_ver)
_dic_link = self.info_result[keyname][last_ver] = {}
for filepath in static_file_lst:
if self._is_disable_suffix(filepath):
continue
md5_string = file_hash(filepath)
web_file = self.web_file_path(filepath)
_dic_link[web_file] = md5_string
def _check(version_compare_lst):
compare_lst = ['>']
# avoid [>= 1.1, <= 1.0]
for compare, _ in version_compare_lst:
compare = compare.strip('=')
if compare != compare_lst[-1]:
compare_lst.append(compare)
length = len(compare_lst)
if version_compare_lst and 0 < length < 3:
return True
logger.warning('maybe framework or cms had be change by developer')
if len(version_compare_lst) < 1:
logger.warning(
'Reducing depth(--depth), set smaller number maybe useful')
logger.error('unable to judge version')
sys.exit()
elif length > 2:
logger.warning(
'Enlarge depth(--depth), set larger number or max(0) maybe useful'
)
lst = [('version cond', )]
for comb, ver in version_compare_lst:
lst.append(('{} {}'.format(comb, ver), ))
show_output(AsciiTable(lst).table)
sys.exit()
def calc(version_compare_set):
"""calcute version compare list."""
def _check(version_compare_lst):
compare_lst = ['>']
# avoid [>= 1.1, <= 1.0]
for compare, _ in version_compare_lst:
compare = compare.strip('=')
if compare != compare_lst[-1]:
compare_lst.append(compare)
length = len(compare_lst)
if version_compare_lst and 0 < length < 3:
return True
logger.warning('maybe framework or cms had be change by developer')
if len(version_compare_lst) < 1:
logger.warning(
'Reducing depth(--depth), set smaller number maybe useful')
logger.error('unable to judge version')
sys.exit()
elif length > 2:
logger.warning(
'Enlarge depth(--depth), set larger number or max(0) maybe useful'
)
lst = [('version cond', )]
for comb, ver in version_compare_lst:
lst.append(('{} {}'.format(comb, ver), ))
show_output(AsciiTable(lst).table)
sys.exit()
lst = list(version_compare_set)
lst.sort(key=cmp_to_key(version_compare_sort))
logger.verbose('compare list after sort: %s', lst)
if _check(lst):
if len(lst) == 1:
show_output(' '.join(lst[0]))
return lst
for prev_, next_ in zip(lst[:-1], lst[1:]):
if prev_[0].strip('=') == '>' \
and next_[0].strip('=') == '<':
lst = [('version cond', ), ('{} {}'.format(*prev_), ),
('{} {}'.format(*next_), )]
show_output(AsciiTable(lst).table)
return [prev_, next_]
if lst[0][0].strip('=') == ">":
res = lst[-1]
else:
res = lst[0]
show_output(' '.join(res))
return (res, )
sys.exit()
def static_hash_map_test():
logger.warning('static_hash_map_test start!')
django = search('django')
distri = file_distribute(django)
logger.info('file_distribute end.')
target_website = 'http://127.0.0.1:8000/'
map_ = static_hash_map(target_website, distri, depth=0)
logger.info(json.dumps(map_, indent=4, sort_keys=True))
logger.info('static_hash_map_test pass!')
def make_diff(self, version_lst, keyname):
"""Comparison between versions one by one"""
for version, prev_version in zip(version_lst[:-1], version_lst[1:]):
next_ver, prev = version.vstring, prev_version.vstring
logger.verbose('make hash dictionary, key: %s, version: %s-%s', keyname, next_ver, prev)
output = self._git_exec('diff', '--name-only', next_ver, prev,
'--', self.static_path, decode=True)
filelst = output.split()
if not filelst:
continue
_dic_link = self.info_result[keyname][prev] = {}
for filename in filelst:
if self._is_disable_suffix(filename):
continue
hash_string = self.ancestor_file(prev, filename)
web_file = self.web_file_path(filename)
_dic_link[web_file] = hash_string
def request_file_hash(url):
"""Return hash string of file by request url."""
logger.verbose('request get url: %s', url)
try:
response = requests.get(url,
verify=False,
allow_redirects=True,
timeout=10,
headers=HTTP_HEADERS)
if response.status_code != 200:
raise requests.RequestException('Status code error: {}'.format(
response.status_code))
except requests.RequestException as ex:
logger.trace('request %s, exception: %s', url, str(ex))
return ''
else:
cont = response.content
return byte_hash(cont)
def reverse_diff(self):
"""Comparison between versions one by one from old to new."""
# version_lst in here must start from min version in fingerprint
fingerprint_versions = self.info_result.get('fingerprint').keys()
min_version = min(
sorted((str2version(vstr) for vstr in fingerprint_versions)))
version_lst = self.version_lst
version_lst.reverse()
logger.warning(version_lst)
version_lst = version_lst[version_lst.index(min_version):]
start_version = version_lst[1]
logger.verbose('the first version with static file is %s',
start_version.vstring)
logger.verbose('reverse version list %s', version_lst)
logger.verbose('git checkout to %s', start_version.vstring)
self._git_exec('checkout', start_version.vstring)
self.last_hash(version_lst[1:], 'reverse_fingerprint')
self.make_diff(version_lst[1:], 'reverse_fingerprint')
logger.verbose('git checkout to HEAD')
self._git_exec('checkout', '-')
def make_version_test():
logger.warning('make_version_test start!')
import ext.err_hunter as err_hunter
from test_data import target_tmp_hash_map3 as target_tmp_hash_map
from observer.version import make_version
django = search('django')
logger.critical(
'make_version return: %s',
make_version(target_tmp_hash_map, django.get('fingerprint')))
logger.critical(
'make_version return: %s',
make_version(target_tmp_hash_map, django.get('reverse_fingerprint'),
False))
logger.info('make_version_test pass!')
def make_version(static_map, fingerprint_map, reverse=True):
"""
return version expression. compare to each version fingerprint.
make compare expressions. like [(">=", 'v2.3.3.3')]
"""
version_compare_set = set()
key_lst = fingerprint_map.keys()
version_lst = sorted([str2version(ver_) for ver_ in key_lst],
reverse=reverse)
# head version in reverse version list is different
head_version_str = version_lst[0].vstring
fingerprint = head_fingerprint = fingerprint_map.get(head_version_str)
match_head = match(static_map, head_fingerprint)
if match_head and reverse:
version_compare_set.add(('>', version_lst[1].vstring))
elif match_head and not reverse:
version_compare_set.add(('>=', head_version_str))
for version in version_lst[1:]:
logger.debug('create operator in version: %s', version.vstring)
fingerprint.update(fingerprint_map.get(version.vstring))
if match(static_map, fingerprint):
operator = OPERATOR_MAP.get(reverse)
version_compare_set.add((operator, version.vstring))
logger.verbose('create version opreator: %s %s', operator,
version.vstring)
logger.debug("operator: %s", version_compare_set)
return version_compare_set
def call_multi_process(function, args_lst, timeout=float('inf')):
"""
use multiprocessing to call the function like pool.map.
:param function: function to call.
:param function: iterable object contains args.
"""
process_pool = init_process_pool()
results = {}
childs = []
start_time = time.time()
for args in args_lst:
def _callback(result, args_=args):
results[args_] = result
if not isinstance(args, (list, tuple, collections.Generator)):
args = (args, )
childs.append(
process_pool.apply_async(function, args, callback=_callback))
try:
while True:
time.sleep(0.5)
now = time.time()
if (now - start_time) > timeout:
raise ProcessTimeoutError(
"main process time cost: {}".format(now))
if all((child.ready() for child in childs)):
break
except (KeyboardInterrupt, ProcessTimeoutError) as ex:
logger.warning("stopping by user interrupt. exception info: %s", ex)
process_pool.terminate()
process_pool.join()
else:
process_pool.close()
process_pool.join()
return results
def make_result(self):
"""Make info_result."""
framework_name = self.framework_name
logger.info('functions has done. init the result and set the name and alias.')
self.info_result['framework'] = framework_name
self.add_alias(framework_name)
logger.info('set the all versions.')
self.set_versions()
logger.info('diff the file version by version and get the info of different file.')
self.forward_diff()
self.reverse_diff()
def ancestor_file(self, vstring, filepath):
"""return the hash of file in one version."""
args = ('show', '{}:{}'.format(vstring, filepath))
try:
output = self._git_exec(*args, decode=False)
except subprocess.CalledProcessError as ex:
logger.noise('this path is not exists on disk in version %s:%s', vstring, filepath)
logger.noise('subprocess error message: %s', ex)
logger.noise('stderr fatal message: %s', ex.output)
return ''
else:
return byte_hash(output)
def mdf_of_file_and_byte_content_test():
logger.warning('mdf_of_file_and_byte_content_test start!')
filepath = './observer.py'
fbyte = open(filepath, 'rb').read()
assert (file_md5(filepath) == byte_md5(fbyte)) is True
logger.info('mdf_of_file_and_byte_content_test pass!')
def show_output(msg):
"""show result."""
logger.critical('result: \n\n%s\n', msg)
def check_run_options(args):
"""url(-u/--url) and frameworks name(-d/--depend) is required."""
if not args.url or not args.depend:
logger.error(
"url(-u/--url) and frameworks name(-d/--depend) is required")
sys.exit()
def run():
"""main function."""
args = call_parser()
check_run_options(args)
depend = args.depend
logger.info('searching %s fingerprint infomation.....', depend)
plugin_info = search(depend)
if not plugin_info:
logger.error('%s can not find a fingerprint of %s', APPNAME, depend)
logger.info('your can use --all to print all fingerprint supported.')
# TODO: show the request fingerprint url in github
sys.exit()
logger.info('already found %s fingerprint.', depend)
distri = file_distribute(plugin_info)
logger.info('start to request hash map on %s in depth %d.', args.url,
args.depth)
hash_map = static_hash_map(args.url, distri, args.depth)
logger.verbose('show the hash map: %s',
json.dumps(hash_map, indent=4, sort_keys=True))
logger.info('let\'s observer which version of %s.', depend)
version_set = make_all(hash_map, plugin_info)
cond_lst = [
VersionCond.from_str(''.join(comp)) for comp in calc(version_set)
]
logger.info('show the possible versions of %s on %s', depend, args.url)
result_lst = [('possible version', )]
for version_str in plugin_info.get('versions'):
if all((cond.match(version_str) for cond in cond_lst)):
info = '{} v{}'.format(depend, version_str)
logger.verbose(info)
result_lst.append((info, ))
show_output(AsciiTable(result_lst).table)
sys.exit(0)
def file_distribute_test():
"""
Return like:
{1: {'/static/admin/admin/js/vendor/jquery/LICENSE-JQUERY.txt'},
2: {'/static/admin/admin/img/sorting-icons.gif',
'/static/admin/admin/img/tooltag-add.svg',
'/static/admin/admin/js/vendor/select2/i18n/gl.js',
'/static/admin/admin/js/vendor/select2/i18n/ru.js',
'/static/admin/admin/img/LICENSE',
'/static/admin/admin/js/vendor/select2/i18n/km.js',
'/static/admin/admin/img/tool-left.gif',
'/static/admin/admin/js/vendor/select2/i18n/da.js',
'/static/admin/admin/img/tooltag-add.png',
'/static/admin/admin/js/vendor/select2/i18n/sv.js',
'/static/admin/admin/img/icon-changelink.svg',
'/static/admin/admin/fonts/Roboto-Bold-webfont.woff',
'/static/admin/admin/img/gis/move_vertex_off.svg',
'/static/admin/admin/img/icon_deletelink.gif',
'/static/admin/admin/img/gis/move_vertex_on.svg',
'/static/admin/admin/img/tooltag-add.gif',
'/static/admin/admin/js/vendor/select2/i18n/es.js',
'/static/admin/admin/js/vendor/select2/i18n/pt.js',
'/static/admin/admin/img/icon-no.gif',
'/static/admin/admin/js/autocomplete.js',
'/static/admin/admin/img/icon_error.gif',
'/static/admin/admin/js/vendor/select2/i18n/pt-BR.js',
'/static/admin/admin/img/chooser_stacked-bg.gif',
'/static/admin/admin/img/icon-unknown.svg',
'/static/admin/admin/js/vendor/select2/select2.full.js',
'/static/admin/admin/js/vendor/select2/LICENSE-SELECT2.md',
'/static/admin/admin/js/vendor/select2/i18n/hi.js',
'/static/admin/admin/img/sorting-icons.svg',
'/static/admin/admin/img/chooser-bg.gif',
'/static/admin/admin/js/getElementsBySelector.js',
'/static/admin/admin/js/vendor/select2/i18n/zh-TW.js',
'/static/admin/admin/img/selector-icons.svg',
'/static/admin/admin/img/tooltag-arrowright.svg',
'/static/admin/admin/js/vendor/select2/i18n/hu.js',
'/static/admin/admin/js/vendor/select2/i18n/sr.js',
'/static/admin/admin/img/icon_clock.gif',
'/static/admin/admin/css/responsive_rtl.css',
'/static/admin/admin/js/vendor/select2/i18n/lv.js',
'/static/admin/admin/img/search.svg',
'/static/admin/admin/img/icon-clock.svg',
'/static/admin/admin/img/deleted-overlay.gif',
'/static/admin/admin/js/vendor/select2/i18n/vi.js',
'/static/admin/admin/js/vendor/select2/i18n/ar.js',
'/static/admin/admin/js/cancel.js',
'/static/admin/admin/css/responsive.css',
'/static/admin/admin/img/tooltag-arrowright.gif',
'/static/admin/admin/js/vendor/select2/i18n/ja.js',
'/static/admin/admin/js/popup_response.js',
'/static/admin/admin/js/vendor/select2/i18n/eu.js',
'/static/admin/admin/css/fonts.css',
'/static/admin/admin/js/vendor/select2/i18n/he.js',
'/static/admin/admin/js/vendor/select2/i18n/ko.js',
'/static/admin/admin/fonts/Roboto-Light-webfont.woff',
'/static/admin/admin/js/vendor/select2/i18n/pl.js',
'/static/admin/admin/js/vendor/jquery/jquery.js',
'/static/admin/admin/img/gis/move_vertex_on.png',
'/static/admin/admin/js/vendor/select2/i18n/ms.js',
'/static/admin/admin/img/icon_calendar.gif',
'/static/admin/admin/img/tooltag-add_over.gif',
'/static/admin/admin/js/vendor/select2/i18n/tr.js',
'/static/admin/admin/js/admin/ordering.js',
'/static/admin/admin/js/vendor/select2/i18n/th.js',
'/static/admin/admin/js/vendor/xregexp/xregexp.min.js',
'/static/admin/admin/css/autocomplete.css',
'/static/admin/admin/img/tool-right.gif',
'/static/admin/admin/js/vendor/select2/i18n/lt.js',
'/static/admin/admin/js/vendor/select2/i18n/az.js',
'/static/admin/admin/img/icon-addlink.svg',
'/static/admin/admin/js/vendor/xregexp/xregexp.js',
'/static/admin/admin/js/LICENSE-JQUERY.txt',
'/static/admin/admin/img/changelist-bg_rtl.gif',
'/static/admin/admin/img/icon-alert.svg',
'/static/admin/admin/fonts/Roboto-Regular-webfont.woff',
'/static/admin/admin/js/vendor/select2/i18n/id.js',
'/static/admin/admin/img/nav-bg-selected.gif',
'/static/admin/admin/js/vendor/select2/i18n/fi.js',
'/static/admin/admin/css/vendor/select2/LICENSE-SELECT2.md',
'/static/admin/admin/js/vendor/select2/i18n/sr-Cyrl.js',
'/static/admin/admin/img/tool-left_over.gif',
'/static/admin/admin/js/vendor/select2/i18n/it.js',
'/static/admin/admin/img/icon-unknown.gif',
'/static/admin/admin/js/compress.py',
'/static/admin/admin/js/vendor/select2/i18n/cs.js',
'/static/admin/admin/img/icon-deletelink.svg',
'/static/admin/admin/img/tool-right_over.gif',
'/static/admin/admin/js/vendor/select2/i18n/ro.js',
'/static/admin/admin/js/vendor/select2/i18n/fa.js',
'/static/admin/admin/img/icon-no.svg',
'/static/admin/admin/js/vendor/select2/i18n/ca.js',
'/static/admin/admin/img/icon_success.gif',
'/static/admin/admin/css/vendor/select2/select2.min.css',
'/static/admin/admin/js/vendor/select2/i18n/bg.js',
'/static/admin/admin/js/vendor/xregexp/LICENSE-XREGEXP.txt',
'/static/admin/admin/img/icon-yes.svg',
'/static/admin/admin/js/vendor/select2/i18n/hr.js',
'/static/admin/admin/js/vendor/select2/i18n/zh-CN.js',
'/static/admin/admin/img/selector-search.gif',
'/static/admin/admin/img/icon-unknown-alt.svg',
'/static/admin/admin/js/vendor/select2/i18n/en.js',
'/static/admin/admin/js/vendor/select2/i18n/nb.js',
'/static/admin/admin/img/nav-bg-grabber.gif',
'/static/admin/admin/js/vendor/select2/i18n/mk.js',
'/static/admin/admin/img/README.txt',
'/static/admin/admin/js/vendor/select2/i18n/sk.js',
'/static/admin/admin/img/gis/move_vertex_off.png',
'/static/admin/admin/img/selector-icons.gif',
'/static/admin/admin/js/vendor/select2/select2.full.min.js',
'/static/admin/admin/img/icon-yes.gif',
'/static/admin/admin/js/prepopulate_init.js',
'/static/admin/admin/js/vendor/select2/i18n/el.js',
'/static/admin/admin/img/icon_addlink.gif',
'/static/admin/admin/img/tooltag-arrowright.png',
'/static/admin/admin/img/calendar-icons.svg',
'/static/admin/admin/js/vendor/select2/i18n/fr.js',
'/static/admin/admin/js/vendor/select2/i18n/uk.js',
'/static/admin/admin/img/tooltag-arrowright_over.gif',
'/static/admin/admin/js/related-widget-wrapper.js',
'/static/admin/admin/img/icon-calendar.svg',
'/static/admin/admin/img/icon_alert.gif',
'/static/admin/admin/css/vendor/select2/select2.css',
'/static/admin/admin/img/inline-delete.svg',
'/static/admin/admin/js/vendor/select2/i18n/is.js',
'/static/admin/admin/js/vendor/select2/i18n/de.js',
'/static/admin/admin/fonts/README.txt',
'/static/admin/admin/js/vendor/select2/i18n/et.js',
'/static/admin/admin/js/vendor/select2/i18n/nl.js',
'/static/admin/admin/img/icon_changelink.gif'},
3: {'/static/admin/admin/img/inline-delete.png',
'/static/admin/admin/js/vendor/jquery/jquery.min.js',
'/static/admin/admin/fonts/LICENSE.txt',
'/static/admin/admin/img/inline-restore-8bit.png',
'/static/admin/admin/js/change_form.js',
'/static/admin/admin/img/changelist-bg.gif',
'/static/admin/admin/img/icon_searchbox.png',
'/static/admin/admin/css/ie.css',
'/static/admin/admin/img/default-bg.gif',
'/static/admin/admin/js/timeparse.js',
'/static/admin/admin/img/inline-restore.png',
'/static/admin/admin/img/inline-delete-8bit.png',
'/static/admin/admin/img/inline-splitter-bg.gif',
'/static/admin/admin/img/nav-bg.gif',
'/static/admin/admin/img/default-bg-reverse.gif',
'/static/admin/admin/img/nav-bg-reverse.gif'},
4: {'/static/admin/admin/js/jquery.init.js',
'/static/admin/admin/css/login.css',
'/static/admin/admin/css/dashboard.css',
'/static/admin/admin/js/prepopulate.js',
'/static/admin/admin/js/jquery.js',
'/static/admin/admin/js/collapse.js',
'/static/admin/admin/js/jquery.min.js'},
5: {'/static/admin/admin/js/prepopulate.min.js',
'/static/admin/admin/js/SelectBox.js'},
6: {'/static/admin/admin/css/changelists.css',
'/static/admin/admin/css/rtl.css',
'/static/admin/admin/js/urlify.js'},
7: {'/static/admin/admin/js/collapse.min.js',
'/static/admin/admin/js/calendar.js'},
8: {'/static/admin/admin/js/actions.min.js',
'/static/admin/admin/js/actions.js',
'/static/admin/admin/js/inlines.js'},
9: {'/static/admin/admin/js/SelectFilter2.js',
'/static/admin/admin/js/inlines.min.js',
'/static/admin/admin/css/forms.css'},
10: {'/static/admin/admin/css/widgets.css',
'/static/admin/admin/js/core.js',
'/static/admin/admin/css/base.css'},
11: {'/static/admin/admin/js/admin/DateTimeShortcuts.js'},
14: {'/static/admin/admin/js/admin/RelatedObjectLookups.js'}}
"""
django = search('django')
distri = file_distribute(django)
logger.info(distri)
