def _git_exec(self, *args, decode=True): """Done 'git' command.""" master_program = 'git' command_tup = (master_program, *args) logger.debug('runing command: %s', ' '.join(command_tup)) output = subprocess.check_output(command_tup, cwd=self.target_project_path, stderr=subprocess.STDOUT) return output.decode() if decode else output
def forward_diff(self): """Comparison between versions one by one from new to old.""" version_lst = self.version_lst self._git_exec('checkout', self.version_lst[0].vstring) logger.verbose('forward version list %s', version_lst) self.last_hash(version_lst, 'fingerprint') self.make_diff(version_lst, 'fingerprint') self._git_exec('checkout', '-')
def show_all(): """show all plugin, --all/-a option.""" logger.info('show all plugin introduction') table_lst = [('framework name', 'tags')] for plugin_info in all_plugin(): tags = ', '.join(plugin_info.get('alias')) table_lst.append((plugin_info.get('framework'), tags)) table = AsciiTable(table_lst) show_output(table.table)
def main(): filter_word = '' try: filter_word = sys.argv[1] except IndexError: pass current_module = sys.modules[__name__] for member, module in inspect.getmembers(current_module): if member.endswith('_test') and filter_word in member: logger.noise("test function: %s, module: %s", member, str(module)) getattr(current_module, member)()
def last_hash(self, version_lst, keyname): """Get all static file's hash string.""" static_file_lst = self.last_static() last_ver = str(version_lst[0]) logger.verbose('last hash dictionary, key: %s, version: %s', keyname, last_ver) _dic_link = self.info_result[keyname][last_ver] = {} for filepath in static_file_lst: if self._is_disable_suffix(filepath): continue md5_string = file_hash(filepath) web_file = self.web_file_path(filepath) _dic_link[web_file] = md5_string
def _check(version_compare_lst): compare_lst = ['>'] # avoid [>= 1.1, <= 1.0] for compare, _ in version_compare_lst: compare = compare.strip('=') if compare != compare_lst[-1]: compare_lst.append(compare) length = len(compare_lst) if version_compare_lst and 0 < length < 3: return True logger.warning('maybe framework or cms had be change by developer') if len(version_compare_lst) < 1: logger.warning( 'Reducing depth(--depth), set smaller number maybe useful') logger.error('unable to judge version') sys.exit() elif length > 2: logger.warning( 'Enlarge depth(--depth), set larger number or max(0) maybe useful' ) lst = [('version cond', )] for comb, ver in version_compare_lst: lst.append(('{} {}'.format(comb, ver), )) show_output(AsciiTable(lst).table) sys.exit()
def calc(version_compare_set): """calcute version compare list.""" def _check(version_compare_lst): compare_lst = ['>'] # avoid [>= 1.1, <= 1.0] for compare, _ in version_compare_lst: compare = compare.strip('=') if compare != compare_lst[-1]: compare_lst.append(compare) length = len(compare_lst) if version_compare_lst and 0 < length < 3: return True logger.warning('maybe framework or cms had be change by developer') if len(version_compare_lst) < 1: logger.warning( 'Reducing depth(--depth), set smaller number maybe useful') logger.error('unable to judge version') sys.exit() elif length > 2: logger.warning( 'Enlarge depth(--depth), set larger number or max(0) maybe useful' ) lst = [('version cond', )] for comb, ver in version_compare_lst: lst.append(('{} {}'.format(comb, ver), )) show_output(AsciiTable(lst).table) sys.exit() lst = list(version_compare_set) lst.sort(key=cmp_to_key(version_compare_sort)) logger.verbose('compare list after sort: %s', lst) if _check(lst): if len(lst) == 1: show_output(' '.join(lst[0])) return lst for prev_, next_ in zip(lst[:-1], lst[1:]): if prev_[0].strip('=') == '>' \ and next_[0].strip('=') == '<': lst = [('version cond', ), ('{} {}'.format(*prev_), ), ('{} {}'.format(*next_), )] show_output(AsciiTable(lst).table) return [prev_, next_] if lst[0][0].strip('=') == ">": res = lst[-1] else: res = lst[0] show_output(' '.join(res)) return (res, ) sys.exit()
def static_hash_map_test(): logger.warning('static_hash_map_test start!') django = search('django') distri = file_distribute(django) logger.info('file_distribute end.') target_website = 'http://127.0.0.1:8000/' map_ = static_hash_map(target_website, distri, depth=0) logger.info(json.dumps(map_, indent=4, sort_keys=True)) logger.info('static_hash_map_test pass!')
def make_diff(self, version_lst, keyname): """Comparison between versions one by one""" for version, prev_version in zip(version_lst[:-1], version_lst[1:]): next_ver, prev = version.vstring, prev_version.vstring logger.verbose('make hash dictionary, key: %s, version: %s-%s', keyname, next_ver, prev) output = self._git_exec('diff', '--name-only', next_ver, prev, '--', self.static_path, decode=True) filelst = output.split() if not filelst: continue _dic_link = self.info_result[keyname][prev] = {} for filename in filelst: if self._is_disable_suffix(filename): continue hash_string = self.ancestor_file(prev, filename) web_file = self.web_file_path(filename) _dic_link[web_file] = hash_string
def request_file_hash(url): """Return hash string of file by request url.""" logger.verbose('request get url: %s', url) try: response = requests.get(url, verify=False, allow_redirects=True, timeout=10, headers=HTTP_HEADERS) if response.status_code != 200: raise requests.RequestException('Status code error: {}'.format( response.status_code)) except requests.RequestException as ex: logger.trace('request %s, exception: %s', url, str(ex)) return '' else: cont = response.content return byte_hash(cont)
def reverse_diff(self): """Comparison between versions one by one from old to new.""" # version_lst in here must start from min version in fingerprint fingerprint_versions = self.info_result.get('fingerprint').keys() min_version = min( sorted((str2version(vstr) for vstr in fingerprint_versions))) version_lst = self.version_lst version_lst.reverse() logger.warning(version_lst) version_lst = version_lst[version_lst.index(min_version):] start_version = version_lst[1] logger.verbose('the first version with static file is %s', start_version.vstring) logger.verbose('reverse version list %s', version_lst) logger.verbose('git checkout to %s', start_version.vstring) self._git_exec('checkout', start_version.vstring) self.last_hash(version_lst[1:], 'reverse_fingerprint') self.make_diff(version_lst[1:], 'reverse_fingerprint') logger.verbose('git checkout to HEAD') self._git_exec('checkout', '-')
def make_version_test(): logger.warning('make_version_test start!') import ext.err_hunter as err_hunter from test_data import target_tmp_hash_map3 as target_tmp_hash_map from observer.version import make_version django = search('django') logger.critical( 'make_version return: %s', make_version(target_tmp_hash_map, django.get('fingerprint'))) logger.critical( 'make_version return: %s', make_version(target_tmp_hash_map, django.get('reverse_fingerprint'), False)) logger.info('make_version_test pass!')
def make_version(static_map, fingerprint_map, reverse=True): """ return version expression. compare to each version fingerprint. make compare expressions. like [(">=", 'v2.3.3.3')] """ version_compare_set = set() key_lst = fingerprint_map.keys() version_lst = sorted([str2version(ver_) for ver_ in key_lst], reverse=reverse) # head version in reverse version list is different head_version_str = version_lst[0].vstring fingerprint = head_fingerprint = fingerprint_map.get(head_version_str) match_head = match(static_map, head_fingerprint) if match_head and reverse: version_compare_set.add(('>', version_lst[1].vstring)) elif match_head and not reverse: version_compare_set.add(('>=', head_version_str)) for version in version_lst[1:]: logger.debug('create operator in version: %s', version.vstring) fingerprint.update(fingerprint_map.get(version.vstring)) if match(static_map, fingerprint): operator = OPERATOR_MAP.get(reverse) version_compare_set.add((operator, version.vstring)) logger.verbose('create version opreator: %s %s', operator, version.vstring) logger.debug("operator: %s", version_compare_set) return version_compare_set
def call_multi_process(function, args_lst, timeout=float('inf')): """ use multiprocessing to call the function like pool.map. :param function: function to call. :param function: iterable object contains args. """ process_pool = init_process_pool() results = {} childs = [] start_time = time.time() for args in args_lst: def _callback(result, args_=args): results[args_] = result if not isinstance(args, (list, tuple, collections.Generator)): args = (args, ) childs.append( process_pool.apply_async(function, args, callback=_callback)) try: while True: time.sleep(0.5) now = time.time() if (now - start_time) > timeout: raise ProcessTimeoutError( "main process time cost: {}".format(now)) if all((child.ready() for child in childs)): break except (KeyboardInterrupt, ProcessTimeoutError) as ex: logger.warning("stopping by user interrupt. exception info: %s", ex) process_pool.terminate() process_pool.join() else: process_pool.close() process_pool.join() return results
def make_result(self): """Make info_result.""" framework_name = self.framework_name logger.info('functions has done. init the result and set the name and alias.') self.info_result['framework'] = framework_name self.add_alias(framework_name) logger.info('set the all versions.') self.set_versions() logger.info('diff the file version by version and get the info of different file.') self.forward_diff() self.reverse_diff()
def ancestor_file(self, vstring, filepath): """return the hash of file in one version.""" args = ('show', '{}:{}'.format(vstring, filepath)) try: output = self._git_exec(*args, decode=False) except subprocess.CalledProcessError as ex: logger.noise('this path is not exists on disk in version %s:%s', vstring, filepath) logger.noise('subprocess error message: %s', ex) logger.noise('stderr fatal message: %s', ex.output) return '' else: return byte_hash(output)
def mdf_of_file_and_byte_content_test(): logger.warning('mdf_of_file_and_byte_content_test start!') filepath = './observer.py' fbyte = open(filepath, 'rb').read() assert (file_md5(filepath) == byte_md5(fbyte)) is True logger.info('mdf_of_file_and_byte_content_test pass!')
def show_output(msg): """show result.""" logger.critical('result: \n\n%s\n', msg)
def check_run_options(args): """url(-u/--url) and frameworks name(-d/--depend) is required.""" if not args.url or not args.depend: logger.error( "url(-u/--url) and frameworks name(-d/--depend) is required") sys.exit()
def run(): """main function.""" args = call_parser() check_run_options(args) depend = args.depend logger.info('searching %s fingerprint infomation.....', depend) plugin_info = search(depend) if not plugin_info: logger.error('%s can not find a fingerprint of %s', APPNAME, depend) logger.info('your can use --all to print all fingerprint supported.') # TODO: show the request fingerprint url in github sys.exit() logger.info('already found %s fingerprint.', depend) distri = file_distribute(plugin_info) logger.info('start to request hash map on %s in depth %d.', args.url, args.depth) hash_map = static_hash_map(args.url, distri, args.depth) logger.verbose('show the hash map: %s', json.dumps(hash_map, indent=4, sort_keys=True)) logger.info('let\'s observer which version of %s.', depend) version_set = make_all(hash_map, plugin_info) cond_lst = [ VersionCond.from_str(''.join(comp)) for comp in calc(version_set) ] logger.info('show the possible versions of %s on %s', depend, args.url) result_lst = [('possible version', )] for version_str in plugin_info.get('versions'): if all((cond.match(version_str) for cond in cond_lst)): info = '{} v{}'.format(depend, version_str) logger.verbose(info) result_lst.append((info, )) show_output(AsciiTable(result_lst).table) sys.exit(0)
def file_distribute_test(): """ Return like: {1: {'/static/admin/admin/js/vendor/jquery/LICENSE-JQUERY.txt'}, 2: {'/static/admin/admin/img/sorting-icons.gif', '/static/admin/admin/img/tooltag-add.svg', '/static/admin/admin/js/vendor/select2/i18n/gl.js', '/static/admin/admin/js/vendor/select2/i18n/ru.js', '/static/admin/admin/img/LICENSE', '/static/admin/admin/js/vendor/select2/i18n/km.js', '/static/admin/admin/img/tool-left.gif', '/static/admin/admin/js/vendor/select2/i18n/da.js', '/static/admin/admin/img/tooltag-add.png', '/static/admin/admin/js/vendor/select2/i18n/sv.js', '/static/admin/admin/img/icon-changelink.svg', '/static/admin/admin/fonts/Roboto-Bold-webfont.woff', '/static/admin/admin/img/gis/move_vertex_off.svg', '/static/admin/admin/img/icon_deletelink.gif', '/static/admin/admin/img/gis/move_vertex_on.svg', '/static/admin/admin/img/tooltag-add.gif', '/static/admin/admin/js/vendor/select2/i18n/es.js', '/static/admin/admin/js/vendor/select2/i18n/pt.js', '/static/admin/admin/img/icon-no.gif', '/static/admin/admin/js/autocomplete.js', '/static/admin/admin/img/icon_error.gif', '/static/admin/admin/js/vendor/select2/i18n/pt-BR.js', '/static/admin/admin/img/chooser_stacked-bg.gif', '/static/admin/admin/img/icon-unknown.svg', '/static/admin/admin/js/vendor/select2/select2.full.js', '/static/admin/admin/js/vendor/select2/LICENSE-SELECT2.md', '/static/admin/admin/js/vendor/select2/i18n/hi.js', '/static/admin/admin/img/sorting-icons.svg', '/static/admin/admin/img/chooser-bg.gif', '/static/admin/admin/js/getElementsBySelector.js', '/static/admin/admin/js/vendor/select2/i18n/zh-TW.js', '/static/admin/admin/img/selector-icons.svg', '/static/admin/admin/img/tooltag-arrowright.svg', '/static/admin/admin/js/vendor/select2/i18n/hu.js', '/static/admin/admin/js/vendor/select2/i18n/sr.js', '/static/admin/admin/img/icon_clock.gif', '/static/admin/admin/css/responsive_rtl.css', '/static/admin/admin/js/vendor/select2/i18n/lv.js', '/static/admin/admin/img/search.svg', '/static/admin/admin/img/icon-clock.svg', '/static/admin/admin/img/deleted-overlay.gif', '/static/admin/admin/js/vendor/select2/i18n/vi.js', '/static/admin/admin/js/vendor/select2/i18n/ar.js', '/static/admin/admin/js/cancel.js', '/static/admin/admin/css/responsive.css', '/static/admin/admin/img/tooltag-arrowright.gif', '/static/admin/admin/js/vendor/select2/i18n/ja.js', '/static/admin/admin/js/popup_response.js', '/static/admin/admin/js/vendor/select2/i18n/eu.js', '/static/admin/admin/css/fonts.css', '/static/admin/admin/js/vendor/select2/i18n/he.js', '/static/admin/admin/js/vendor/select2/i18n/ko.js', '/static/admin/admin/fonts/Roboto-Light-webfont.woff', '/static/admin/admin/js/vendor/select2/i18n/pl.js', '/static/admin/admin/js/vendor/jquery/jquery.js', '/static/admin/admin/img/gis/move_vertex_on.png', '/static/admin/admin/js/vendor/select2/i18n/ms.js', '/static/admin/admin/img/icon_calendar.gif', '/static/admin/admin/img/tooltag-add_over.gif', '/static/admin/admin/js/vendor/select2/i18n/tr.js', '/static/admin/admin/js/admin/ordering.js', '/static/admin/admin/js/vendor/select2/i18n/th.js', '/static/admin/admin/js/vendor/xregexp/xregexp.min.js', '/static/admin/admin/css/autocomplete.css', '/static/admin/admin/img/tool-right.gif', '/static/admin/admin/js/vendor/select2/i18n/lt.js', '/static/admin/admin/js/vendor/select2/i18n/az.js', '/static/admin/admin/img/icon-addlink.svg', '/static/admin/admin/js/vendor/xregexp/xregexp.js', '/static/admin/admin/js/LICENSE-JQUERY.txt', '/static/admin/admin/img/changelist-bg_rtl.gif', '/static/admin/admin/img/icon-alert.svg', '/static/admin/admin/fonts/Roboto-Regular-webfont.woff', '/static/admin/admin/js/vendor/select2/i18n/id.js', '/static/admin/admin/img/nav-bg-selected.gif', '/static/admin/admin/js/vendor/select2/i18n/fi.js', '/static/admin/admin/css/vendor/select2/LICENSE-SELECT2.md', '/static/admin/admin/js/vendor/select2/i18n/sr-Cyrl.js', '/static/admin/admin/img/tool-left_over.gif', '/static/admin/admin/js/vendor/select2/i18n/it.js', '/static/admin/admin/img/icon-unknown.gif', '/static/admin/admin/js/compress.py', '/static/admin/admin/js/vendor/select2/i18n/cs.js', '/static/admin/admin/img/icon-deletelink.svg', '/static/admin/admin/img/tool-right_over.gif', '/static/admin/admin/js/vendor/select2/i18n/ro.js', '/static/admin/admin/js/vendor/select2/i18n/fa.js', '/static/admin/admin/img/icon-no.svg', '/static/admin/admin/js/vendor/select2/i18n/ca.js', '/static/admin/admin/img/icon_success.gif', '/static/admin/admin/css/vendor/select2/select2.min.css', '/static/admin/admin/js/vendor/select2/i18n/bg.js', '/static/admin/admin/js/vendor/xregexp/LICENSE-XREGEXP.txt', '/static/admin/admin/img/icon-yes.svg', '/static/admin/admin/js/vendor/select2/i18n/hr.js', '/static/admin/admin/js/vendor/select2/i18n/zh-CN.js', '/static/admin/admin/img/selector-search.gif', '/static/admin/admin/img/icon-unknown-alt.svg', '/static/admin/admin/js/vendor/select2/i18n/en.js', '/static/admin/admin/js/vendor/select2/i18n/nb.js', '/static/admin/admin/img/nav-bg-grabber.gif', '/static/admin/admin/js/vendor/select2/i18n/mk.js', '/static/admin/admin/img/README.txt', '/static/admin/admin/js/vendor/select2/i18n/sk.js', '/static/admin/admin/img/gis/move_vertex_off.png', '/static/admin/admin/img/selector-icons.gif', '/static/admin/admin/js/vendor/select2/select2.full.min.js', '/static/admin/admin/img/icon-yes.gif', '/static/admin/admin/js/prepopulate_init.js', '/static/admin/admin/js/vendor/select2/i18n/el.js', '/static/admin/admin/img/icon_addlink.gif', '/static/admin/admin/img/tooltag-arrowright.png', '/static/admin/admin/img/calendar-icons.svg', '/static/admin/admin/js/vendor/select2/i18n/fr.js', '/static/admin/admin/js/vendor/select2/i18n/uk.js', '/static/admin/admin/img/tooltag-arrowright_over.gif', '/static/admin/admin/js/related-widget-wrapper.js', '/static/admin/admin/img/icon-calendar.svg', '/static/admin/admin/img/icon_alert.gif', '/static/admin/admin/css/vendor/select2/select2.css', '/static/admin/admin/img/inline-delete.svg', '/static/admin/admin/js/vendor/select2/i18n/is.js', '/static/admin/admin/js/vendor/select2/i18n/de.js', '/static/admin/admin/fonts/README.txt', '/static/admin/admin/js/vendor/select2/i18n/et.js', '/static/admin/admin/js/vendor/select2/i18n/nl.js', '/static/admin/admin/img/icon_changelink.gif'}, 3: {'/static/admin/admin/img/inline-delete.png', '/static/admin/admin/js/vendor/jquery/jquery.min.js', '/static/admin/admin/fonts/LICENSE.txt', '/static/admin/admin/img/inline-restore-8bit.png', '/static/admin/admin/js/change_form.js', '/static/admin/admin/img/changelist-bg.gif', '/static/admin/admin/img/icon_searchbox.png', '/static/admin/admin/css/ie.css', '/static/admin/admin/img/default-bg.gif', '/static/admin/admin/js/timeparse.js', '/static/admin/admin/img/inline-restore.png', '/static/admin/admin/img/inline-delete-8bit.png', '/static/admin/admin/img/inline-splitter-bg.gif', '/static/admin/admin/img/nav-bg.gif', '/static/admin/admin/img/default-bg-reverse.gif', '/static/admin/admin/img/nav-bg-reverse.gif'}, 4: {'/static/admin/admin/js/jquery.init.js', '/static/admin/admin/css/login.css', '/static/admin/admin/css/dashboard.css', '/static/admin/admin/js/prepopulate.js', '/static/admin/admin/js/jquery.js', '/static/admin/admin/js/collapse.js', '/static/admin/admin/js/jquery.min.js'}, 5: {'/static/admin/admin/js/prepopulate.min.js', '/static/admin/admin/js/SelectBox.js'}, 6: {'/static/admin/admin/css/changelists.css', '/static/admin/admin/css/rtl.css', '/static/admin/admin/js/urlify.js'}, 7: {'/static/admin/admin/js/collapse.min.js', '/static/admin/admin/js/calendar.js'}, 8: {'/static/admin/admin/js/actions.min.js', '/static/admin/admin/js/actions.js', '/static/admin/admin/js/inlines.js'}, 9: {'/static/admin/admin/js/SelectFilter2.js', '/static/admin/admin/js/inlines.min.js', '/static/admin/admin/css/forms.css'}, 10: {'/static/admin/admin/css/widgets.css', '/static/admin/admin/js/core.js', '/static/admin/admin/css/base.css'}, 11: {'/static/admin/admin/js/admin/DateTimeShortcuts.js'}, 14: {'/static/admin/admin/js/admin/RelatedObjectLookups.js'}} """ django = search('django') distri = file_distribute(django) logger.info(distri)