def insertStuff(self, pre, after, preTwo, afterTwo, type): sels = self.view.sel() nSel = 0 for sel in sels: nSel += 1 for sel in sels: if sel.empty(): continue str = self.view.substr(sel) tab = self.insert_start_line(sel) opts = jsbeautifier.default_options() opts.indent_size = 4; if nSel == 1: str = self.insert_tab_line(str) str = str.replace('\n'+tab, '\n') str = pre + '\n\t' + str + '\n' + after str = self.normalize_line_endings(str) str = jsbeautifier.beautify(str, opts) self.view.run_command('insert_snippet', {"contents": str}) else: str = self.insert_tab_line(str) str = preTwo + '\n\t' + tab + str + '\n' + tab + afterTwo str = self.normalize_line_endings(str) str = jsbeautifier.beautify(str, opts) self.view.replace(self.edit, sel, str) sublime.status_message(type + ' added')
def load_settings(): settings_path = join(dirname(realpath(__file__)), "settings.json") try: settings = loads(open(settings_path).read()) has_change = False for i, item in enumerate(settings["processes"]): for key, val in {"process_name": "input_process_name_here", "grep_string": "grep_string", "owner": None, "class": "default"}.items(): if key not in item: settings["processes"][i][key] = val has_change = True for item in settings["processes"]: if item["class"] not in settings["classes"]: settings[item["class"]] = settings["classes"]["default"].copy() has_change = True for prio_class, config in settings["classes"].items(): if "prio_class" not in config or "prio_data" not in config or len(config) != 2: settings["classes"][prio_class].clear() settings["classes"][prio_class]["prio_class"] = settings["classes"]["default"]["prio_class"] settings["classes"][prio_class]["prio_data"] = settings["classes"]["default"]["prio_data"] has_change = True if has_change: open(settings_path, "w+").write(beautify(dumps(settings))) except FileNotFoundError or ValueError: settings = IOVeryNice.default_settings.copy() open(settings_path, "w+").write(beautify(dumps(settings))) return settings
def find_tracts_with_users(): with open('data/' + my.DATA_FOLDER + 'user_homes.json', 'rb') as fp: homes = anyjson.loads(fp.read()) homes = dict( ( int(h[0]), ogr.CreateGeometryFromWkt('POINT(%s %s)' \ % (h[1][1], h[1][0])) ) for h in homes.items() ) with open('data/' + my.DATA_FOLDER + 'city_tracts.json', 'rb') as fp: polygons = anyjson.loads(fp.read()) user_counts = dict( (gid, 0) for gid in polygons.keys()) users_in_tracts = [] tracts_with_users = [] tract_users = dict( (gid, []) for gid in polygons.keys()) for gid, json_poly in polygons.iteritems(): json_poly_str = anyjson.dumps(json_poly) poly = ogr.CreateGeometryFromJson(json_poly_str) for user_id, h in homes.items(): if h.Within(poly): user_counts[gid] += 1 users_in_tracts.append(user_id) tract_users[gid].append(user_id) del homes[user_id] tracts_with_users = [gid for gid, v in user_counts.items() if v != 0] with open('data/' + my.DATA_FOLDER + 'user_counts.json', 'wb') as fp: fp.write( jsb.beautify( anyjson.dumps(user_counts) ) ) with open('data/' + my.DATA_FOLDER + 'tract_users.json', 'wb') as fp: fp.write( jsb.beautify( anyjson.dumps(tract_users) ) ) with open('data/' + my.DATA_FOLDER + 'users_in_tracts.json', 'wb') as fp: fp.write( anyjson.dumps(users_in_tracts) ) with open('data/' + my.DATA_FOLDER + 'tracts_with_users.json', 'wb') as fp: fp.write( anyjson.dumps(tracts_with_users) )
def do_jsbeautify(self,line): try: import jsbeautifier l = line.split(" ") if len(l) < 2: self.help_jsbeautify() else: OPTIONS = ['slice','obj'] option = l[0] if option not in OPTIONS: print "Invalid option" return False id = l[1] response, size = CTCore.get_response_and_size(id, "all") name = CTCore.get_name(id) if option == "slice": offset = int(l[2]) length = l[3] bytes, length = get_bytes(response,offset,length) js_bytes = bytes res = jsbeautifier.beautify(js_bytes) print res if option == "obj": res = jsbeautifier.beautify(response) obj_num = CTCore.add_object("jsbeautify",res,id=id) print " JavaScript Beautify of object {} ({}) successful!".format(str(id), name) print " New object created: {}".format(obj_num) + newLine except Exception,e: print str(e)
def decodesto(self, input, expectation=None): self.assertEqual( jsbeautifier.beautify(input, self.options), expectation or input) # if the expected is different from input, run it again # expected output should be unchanged when run twice. if not expectation == None: self.assertEqual( jsbeautifier.beautify(expectation, self.options), expectation)
def show_campaign(campaign_id): campaign = CampaignController.get(campaign_id) beautified_text = {} if campaign != None: beautified_text = { 'kernel' : jsbeautifier.beautify(campaign.kernel), 'static_dataset' : jsbeautifier.beautify(campaign.static_dataset), 'dataset_gen' : jsbeautifier.beautify(campaign.dataset_gen) } return render_template('/campaign/show_campaign.html', campaign=campaign, beautified_text=beautified_text)
def _make_volume_mat(id): '''''' print '\n', id, '\n' volume_mat = {} with open('data/' + my.DATA_FOLDER + 'sets.json', 'rb') as fp: sets = anyjson.loads(fp.read()).items() with open('data/' + my.DATA_FOLDER + 'time.json', 'rb') as fp: time = anyjson.loads(fp.read()) ts_start = datetime.strptime(time['ts_start'], my.TS_FORMAT) ts_end = datetime.strptime(time['ts_end'], my.TS_FORMAT) diff = ts_end - ts_start diff = int(diff.total_seconds()) combine = 120 x, x_ticks, x_tickslabels = _make_x(ts_start, diff, combine=combine) for pid, st in sets: offsets = _get_offsets(id, st) print pid, st, len(offsets) y = dict((i, 0) for i in x) for o in offsets: idx = int(o/combine) if idx in y: y[idx] += 1 y = y.values() volume_mat[pid] = y path = 'data/' + my.DATA_FOLDER + 'volume_mat/' if not os.path.exists(path): os.makedirs(path) with open(path + str(id) + '.json', 'wb') as fp: fp.write(jsb.beautify(anyjson.dumps(volume_mat)))
def beautifier(self, buffer): 'clean up the JS' try: temp = jsbeautifier.beautify(buffer.read()) except Exception, e: logging.error("ERROR: jsbeautifier: {}".format(e)) return
def test_fragment(input, expectation = None): global opts if expectation == None: expectation = input res = beautify(input, opts) if res != expectation: print("""=== TEST FAILED === %s Input: ------ %s Expected: --------- %s Received: --------- %s """ % (opts, input, expectation, res)) sys.exit(-1) # bail out immediately global tests_passed tests_passed += 1
def handle_javascript(self, script): try: log.info(jsbeautifier.beautify(str(script))) except: log.info(script) #if isinstance(script, BeautifulSoap.Tag): # js = ' '.join(script.contents) #else: # js = script.text js = getattr(script, 'text', None) relationship = 'Contained_Inside' if not js: src = script.get('src', None) if not src: return try: response, js = self.window._navigator.fetch(src) except: return if response.status == 404: return relationship = 'External' if len(js): log.ThugLogging.add_code_snippet(js, 'Javascript', relationship) self.window.evalScript(js, tag = script)
def run(): """ Start program """ options = Options() for file in options.get_files(): if not os.path.isfile(file): raise SystemExit( sys.argv[0] + ': Cannot find "' + file + '" file.') print('Re-formatting "' + file + '" XML file...') lines = [] try: with open(file, errors='replace') as ifile: for line in ifile: lines.append(line.strip()) except OSError: raise SystemExit( sys.argv[0] + ': Cannot read "' + file + '" file.') tmpfile = file + '-tmp' + str(os.getpid()) try: with open(tmpfile, 'w', newline='\n') as ofile: print(jsbeautifier.beautify(''.join(lines)), file=ofile) except OSError: raise SystemExit( sys.argv[0] + ': Cannot create "' + tmpfile + '" file.') try: shutil.move(tmpfile, file) except OSError: raise SystemExit( sys.argv[0] + ': Cannot rename "' + tmpfile + '" file to "' + file + '".' )
def find_user_race(): with open('data/' + my.DATA_FOLDER + 'user_homes.json', 'rb') as fp: homes = anyjson.loads(fp.read()) global loc_data with open('data/' + my.DATA_FOLDER + 'loc_data.json', 'rb') as fp: loc_data = anyjson.loads(fp.read()) #for user in homes.items()[:100]: # _find_race(user) pool = Pool(my.PROCESSES) user_race = pool.map(_find_race, homes.items()) user_race = filter(None, user_race) races = { 'w' : [], 'b' : [], 'a' : [], 'h' : [], 'o' : [] } for user_id, race in user_race: races[race].append(user_id) with open('data/' + my.DATA_FOLDER + 'user_race.json', 'wb') as fp: fp.write(jsb.beautify(anyjson.dumps(races)))
def toEntryPoint(pathJs): firstLine = True newContents = "" contents = source.GetFileContents(pathJs) source_lines = contents.splitlines() for line in source_lines: if _REGEX_GOOG.search(line): continue match = _BASE_REGEX_STRING.match(line) if match: if not firstLine: newContents += ",\n" else: newContents += '\n_moduleConfig={alias:{' newContents += '"' + match.group(2) + '":"' + getAmdFilePath(match.group(1)) + '"' firstLine = False else: newContents += "\n" + line newContents += '\n}};' #输出 out = open(getEntryPointFilePath(pathJs), 'w') beautifier_options = jsbeautifier.BeautifierOptions() beautifier_options.indent_with_tabs = True out.write(jsbeautifier.beautify(newContents, beautifier_options))
def run(): base = open('Processing.hub').read() # find all the $sections sections = re.findall('\$\w+', base) subs = dict(zip(sections, [""]*len(sections))) print subs xml = ET.parse("Serial-Processing.xml") # apply the core elements to the template core = xml.find('core') for element in core: section = "$" + element.tag # remove extra newlines and whitespace content = element.text.strip() # set counters and addresses content = content.replace("$count", "") # add this code to the right section if section in subs: subs[section] += content # substitute the section labels with the code for sub, code in subs.iteritems(): base = base.replace(sub, code) # format // prettify result = jsbeautifier.beautify(base) print result
def run(self, edit): settings = self.view.settings() # settings opts = jsbeautifier.default_options() opts.indent_char = " " if settings.get("translate_tabs_to_spaces") else "\t" opts.indent_size = int(settings.get("tab_size")) if opts.indent_char == " " else 1 opts.max_preserve_newlines = s.get("max_preserve_newlines") or 3 opts.preserve_newlines = s.get("preserve_newlines") or True opts.jslint_happy = s.get("jslint_happy") or False opts.brace_style = s.get("brace_style") or "collapse" opts.s = s.get("keep_array_indentation") or False opts.indent_level = s.get("indent_level") or 0 selection = self.view.sel()[0] nwsOffset = self.prev_non_whitespace() # do formatting and replacement replaceRegion = selection if len(selection) > 0 else sublime.Region(0, self.view.size()) res = jsbeautifier.beautify(self.view.substr(replaceRegion), opts) self.view.replace(edit, replaceRegion, res) # re-place cursor offset = self.get_nws_offset(nwsOffset, self.view.substr(sublime.Region(0, self.view.size()))) rc = self.view.rowcol(offset) pt = self.view.text_point(rc[0], rc[1]) sel = self.view.sel() sel.clear() self.view.sel().add(sublime.Region(pt)) self.view.show_at_center(pt)
def dev_norm(): with open('data/' + my.DATA_FOLDER + 'X.json', 'rb') as fp: X = anyjson.loads(fp.read()) AVG = [0]*len(X[0]) for x in X: for i in range(len(x)): AVG[i] += x[i] for i in range(len(AVG)): AVG[i] /= len(X) X_ = [] for x in X: for i in range(len(x)): x[i] -= AVG[i] mn = min([i for i in x if i < 0]) if mn != 0: mn = mn*(-1) for i in range(len(x)): if x[i] != 0: #x[i] += mn x[i] = abs(x[i]) X_.append(x) with open('data/' + my.DATA_FOLDER + 'X.json', 'wb') as fp: fp.write(jsb.beautify(anyjson.dumps(X_)))
def PLAYVIDEO(url, name): progress.create('Play video', 'Searching videofile.') progress.update( 10, "", "Loading video page", "" ) hosts = [] videosource = getHtml(url, url) if re.search('videomega', videosource, re.DOTALL | re.IGNORECASE): hosts.append('VideoMega') if re.search('openload', videosource, re.DOTALL | re.IGNORECASE): hosts.append('OpenLoad') if re.search('www.flashx.tv', videosource, re.DOTALL | re.IGNORECASE): hosts.append('FlashX') if len(hosts) == 0: progress.close() dialog.ok('Oh oh','Couldn\'t find any video') return elif len(hosts) > 1: if addon.getSetting("dontask") == "true": vidhost = hosts[0] else: vh = dialog.select('Videohost:', hosts) vidhost = hosts[vh] else: vidhost = hosts[0] if vidhost == 'VideoMega': progress.update( 40, "", "Loading videomegatv", "" ) if re.search("http://videomega.tv/iframe.js", videosource, re.DOTALL | re.IGNORECASE): hashref = re.compile("""javascript["']>ref=['"]([^'"]+)""", re.DOTALL | re.IGNORECASE).findall(videosource) else: hashkey = re.compile("""hashkey=([^"']+)""", re.DOTALL | re.IGNORECASE).findall(videosource) hashpage = getHtml('http://videomega.tv/validatehash.php?hashkey='+hashkey[0], url) hashref = re.compile('ref="([^"]+)', re.DOTALL | re.IGNORECASE).findall(hashpage) progress.update( 80, "", "Getting video file", "" ) videopage = getHtml('http://videomega.tv/view.php?ref='+hashref[0], url) videourl = re.compile('<source src="([^"]+)"', re.DOTALL | re.IGNORECASE).findall(videopage) videourl = videourl[0] elif vidhost == 'OpenLoad': progress.update( 40, "", "Loading Openload", "" ) openloadurl = re.compile('<iframe src="([^"]+)"', re.DOTALL | re.IGNORECASE).findall(videosource) openloadsrc = getHtml(openloadurl[0], url) videourl = re.compile('<source src="([^"]+)"', re.DOTALL | re.IGNORECASE).findall(openloadsrc) progress.update( 80, "", "Getting video file", "" ) openload302 = getVideoLink(videourl[0],openloadurl[0]) realurl = openload302.replace('https://','http://') videourl = realurl + "|" + openloadurl[0] elif vidhost == 'FlashX': progress.update( 40, "", "Loading FlashX", "" ) flashxurl = re.compile('<iframe src="([^"]+)"', re.DOTALL | re.IGNORECASE).findall(videosource) flashxsrc = getHtml2(flashxurl[0]) flashxjs = re.compile("<script type='text/javascript'>([^<]+)</sc", re.DOTALL | re.IGNORECASE).findall(flashxsrc) progress.update( 80, "", "Getting video file", "" ) flashxujs = beautify(flashxjs[0]) videourl = re.compile(r',.*file: "([^"]+)".*\}\],', re.DOTALL | re.IGNORECASE).findall(flashxujs) videourl = videourl[0] progress.close() iconimage = xbmc.getInfoImage("ListItem.Thumb") listitem = xbmcgui.ListItem(name, iconImage="DefaultVideo.png", thumbnailImage=iconimage) listitem.setInfo('video', {'Title': name, 'Genre': 'P**n'}) xbmc.Player().play(videourl, listitem)
def get_code_via_id(root,id): selector = ''.join(('//code[@id','=','"',id,'"',']')) eles = root.xpath(selector) children = eles[0].getchildren() comment = children[0] rslt_dict = json.loads(comment.text) print(jb.beautify(comment.text)) return(rslt_dict)
def wutu_js(): if minify: from jsmin import jsmin jsdata = jsmin(get_data(api.jsstream)) else: from jsbeautifier import beautify jsdata = beautify(get_data(api.jsstream)) return Response(jsdata, mimetype="text/javascript")
def format_whole_file(self, edit, opts): view = self.view region = sublime.Region(0, view.size()) code = view.substr(region) formatted_code = jsbeautifier.beautify(code, opts) _, err = merge_utils.merge_code(view, edit, code, formatted_code) if err: sublime.error_message("JsFormat: Merge failure: '%s'" % err)
def prettystringjs(self, str): opts = jsbeautifier.default_options() opts.max_preserve_newlines = 1 res = jsbeautifier.beautify(str) return res
def writeJsx(appdef, folder, app, progress): layers = appdef["Layers"] viewCrs = appdef["Settings"]["App view CRS"] mapbounds = bounds(appdef["Settings"]["Extent"] == "Canvas extent", layers, viewCrs) mapextent = "extent: %s," % mapbounds if appdef["Settings"]["Restrict to extent"] else "" maxZoom = int(appdef["Settings"]["Max zoom level"]) minZoom = int(appdef["Settings"]["Min zoom level"]) app.variables.append("var view = new ol.View({%s maxZoom: %d, minZoom: %d, projection: '%s'});" % (mapextent, maxZoom, minZoom, viewCrs)) app.variables.append("var originalExtent = %s;" % mapbounds) permalink = str(appdef["Settings"]["Add permalink functionality"]).lower() logoImg = appdef["Settings"]["Logo"].strip() logoOption = "" if logoImg: ext = os.path.splitext(logoImg)[1] shutil.copyfile(logoImg, os.path.join(folder, "logo" + ext)) logoOption = ', iconElementLeft: React.createElement("img", {className:"pull-left", style:{margin:"5px",height:"50px"}, src:"logo%s"})' % ext toolbarOptions = '{style:{height: 71}, title:"%s"%s}' % (appdef["Settings"]["Title"], logoOption) app.mappanels.append('''React.createElement("div", {id: 'popup', className: 'ol-popup'}, React.createElement(InfoPopup, {map: map, hover: %s}) )''' % str(appdef["Settings"]["Show popups on hover"]).lower()) variables ="\n".join(app.variables) try: variables = jsbeautifier.beautify(variables) except: pass #jsbeautifier gives some random errors sometimes due to imports def join(array): if array: return ",\n" + ",\n".join(array) else: return "" values = {"@IMPORTS@": "\n".join(app.imports), "@TABS@": join(app.tabs), "@OL3CONTROLS@": ",\n".join(app.ol3controls), "@PANELS@": join(app.panels), "@MAPPANELS@": join(app.mappanels), "@TOOLBAR@": ",\n".join(app.tools), "@TOOLBAROPTIONS@": toolbarOptions, "@VARIABLES@": variables, "@POSTTARGETSET@": "\n".join(app.posttarget), "@PERMALINK@": permalink} template = os.path.join(os.path.dirname(__file__), "themes", appdef["Settings"]["Theme"], "app.jsx") jsx = replaceInTemplate(template, values) name = "app.jsx" jsxFilepath = os.path.join(folder, name) with codecs.open(jsxFilepath, "w", encoding="utf-8") as f: f.write(jsx)
def test_str(str, expected): global fails res = jsbeautifier.beautify(str, opts) if(res == expected): print(".") return True else: print("___got:" + res + "\n___expected:" + expected + "\n") fails = fails + 1 return False
def spatial_norm(): with open('data/' + my.DATA_FOLDER + 'X.json', 'rb') as fp: X = anyjson.loads(fp.read()) X_ = [] for x in X: X_.append(_snorm(x)) with open('data/' + my.DATA_FOLDER + 'X.json', 'wb') as fp: fp.write(jsb.beautify(anyjson.dumps(X_)))
def plot_grid(): '''Plot all geo-tagged tweets on map''' lat1, lng1, lat2, lng2 = my.BBOX xticks = np.arange(lng1, lng2, my.LNG_DELTA).tolist() xticks.append(lng2) print xticks yticks = np.arange(lat1, lat2 + my.LAT_DELTA, my.LAT_DELTA).tolist() fig=plt.figure(figsize=(18,13)) fig.set_tight_layout(True) ax=fig.add_subplot(111) m = Basemap(llcrnrlon=lng1, llcrnrlat=lat1, urcrnrlon=lng2, urcrnrlat=lat2, projection='mill') ax.set_xlim(lng1, lng2) ax.set_ylim(lat1, lat2) ax.set_xticks(xticks) ax.set_yticks(yticks) #ax.grid(ls='-') ax.grid(ls='--', lw=1.25) plt.setp(plt.xticks()[1], rotation=90) bg = matplotlib.image.imread('data/' + my.DATA_FOLDER + 'map.png') ax.imshow(bg, aspect='auto', extent=(lng1, lng2, lat1, lat2), alpha=0.9) plt.savefig('data/' + my.DATA_FOLDER + 'grid' + '.png') yticks.reverse() grid = { 'bbox' : my.BBOX, 'lat_delta' : my.LAT_DELTA, 'lng_delta' : my.LNG_DELTA, 'xticks' : [round(i, 3) for i in xticks], 'yticks' : [round(i, 3) for i in yticks], 'rows' : len(yticks) - 1, 'columns' : len(xticks) - 1, 'cells' : (len(yticks) - 1) * (len(xticks) - 1), 'grid' : {}, #'grid_lookup' : {} } i = 0 for r in range(len(yticks) - 1): #grid['grid_lookup'][round(yticks[r+1], 3)] = {} for c in range(len(xticks) - 1): grid['grid'][i] = ( round(yticks[r+1], 3), round(xticks[c], 3), round(yticks[r], 3), round(xticks[c+1], 3)) #grid['grid_lookup'][round(yticks[r+1], 3)][round(xticks[c], 3)] = i i += 1 with open('data/' + my.DATA_FOLDER + 'grid.json', 'wb') as fp: fp.write(jsb.beautify(anyjson.dumps(grid)))
def eval(self, script): try: log.info(jsbeautifier.beautify(script)) except: log.info(script) if len(script) > 64: log.warning("[Window] Eval argument length > 64 (%d)" % (len(script), )) log.ThugLogging.add_code_snippet(script, 'Javascript', 'Dynamically_Evaluated') return self.evalScript(script)
def run(self, edit): opts = jsbeautifier.default_options(); opts.indent_char = "\t" opts.indent_size = 1 opts.max_preserve_newlines = 3 selection = self.view.sel()[0] replaceRegion = selection if len(selection) > 0 else sublime.Region(0, self.view.size()) res = jsbeautifier.beautify(self.view.substr(replaceRegion), opts) prePos = self.view.sel()[0] self.view.replace(edit, replaceRegion, res) self.view.show_at_center(prePos.begin())
def male_X(): '''''' with open('data/' + my.DATA_FOLDER + 'dates.json', 'rb') as fp: dates = anyjson.loads(fp.read()) path = 'data/' + my.DATA_FOLDER + 'grid_data/' X = [] index = [] for date in dates: with open(path + date + '.json', 'rb') as fp: data = anyjson.loads(fp.read()) data = list(itertools.chain(*data)) X.append(data) index.append(date) with open('data/' + my.DATA_FOLDER + 'X.json', 'wb') as fp: fp.write(jsb.beautify(anyjson.dumps(X))) with open('data/' + my.DATA_FOLDER + 'X_index.json', 'wb') as fp: fp.write(jsb.beautify(anyjson.dumps(index)))
def jsbeautify(javascript): """Beautifies Javascript through jsbeautifier and ignore some messages.""" with _jsbeautify_lock: origout, sys.stdout = sys.stdout, io.StringIO() javascript = jsbeautifier.beautify(javascript) if sys.stdout.getvalue() not in _jsbeautify_blacklist: log.warning("jsbeautifier returned error: %s", sys.stdout.getvalue()) sys.stdout = origout return javascript
def _write(file, script): lines = jsbeautifier.beautify(script).splitlines() print('Writing "{0:s}" with {1:d} lines...'.format(file, len(lines))) try: with open(file, 'w') as ofile: for line in lines: print(line, file=ofile) except OSError: raise SystemExit( sys.argv[0] + ': Cannot write "' + file + '" configuration file.')
def general_code_analysis(paths): """Static Code Analysis""" try: scan_results = {} scan_rules = read_rules() security_issues = [] good_finding = [] missing_header = [] all_files = [] # Initializing Security Header flag as not present header_found = {} sec_issues_by_tag = {} good_finding_by_tag = {} missing_header_by_tag = {} vuln_n_count = {} count = {} # Sort By Tag tags = {'rce': 'Remote Command Execution', 'rci': 'Remote Code Injection', 'ssrf': 'Server Side Request Forgery', 'module': 'Vulnerable Node Module', 'node': 'Application Related', 'web': 'Web Security', 'dir': 'Directory Traversal', 'opr': 'Open Redirect Vulnerability', 'sqli': 'SQL Injection (SQLi)', 'xss': 'Cross Site Scripting (XSS)', 'nosqli': 'NoSQL Injection', 'hhi': 'HTTP Header Injection', } for path in paths: for header in scan_rules["missing_sec_header"].iterkeys(): header_found[header] = 0 print "\n[INFO] Running Static Analyzer Running on - " + path + "\n" for root, _, files in os.walk(path): for filename in files: full_file_path = os.path.join(root, filename) relative_path = full_file_path.replace(path, "") all_files.append({relative_path.replace( "/", "", 1): full_file_path.replace(settings.UPLOAD_FOLDER, "", 1)}) data = is_valid_node(filename, full_file_path) if data is not None: # print relative_path beautified_data = None lines = data.splitlines() if len(lines) <= 2: # Possible Minified Single Line Code try: # Try Because of Possible Bug in JS Beautifier beautified_data = jsbeautifier.beautify(data) lines = beautified_data.splitlines() except: pass for line_no, line in enumerate(lines): # Avoid Comments - Multiline is a problem still if not line.lstrip().startswith("//") and not line.lstrip().startswith("/*"): # Limit the no of caracters in a line to 2000 line = line[0:2000] # Vulnerability String Match for rule_key in scan_rules["vuln_rules"].iterkeys(): if scan_rules["vuln_rules"][rule_key] in line: finding = {} finding["title"] = rule_key finding["description"] = scan_rules[ "desc"][rule_key] finding["tag"] = scan_rules[ "tag"][rule_key] finding["line"] = line_no + 1 finding["lines"] = get_lines( line_no, lines) finding["filename"] = filename finding["path"] = full_file_path.replace( settings.UPLOAD_FOLDER, "", 1) finding["sha2"] = utils.gen_sha256_hash( finding["lines"].encode(encoding="utf-8", errors="replace")) security_issues.append(finding) # Vulnerability Regex Match for regex in scan_rules["vuln_regex"].iterkeys(): if re.search(scan_rules["vuln_regex"][regex], line): finding = {} finding["title"] = regex finding["description"] = scan_rules[ "desc"][regex] finding["tag"] = scan_rules[ "tag"][regex] finding["line"] = line_no + 1 finding["lines"] = get_lines( line_no, lines) finding["filename"] = filename finding["path"] = full_file_path.replace( settings.UPLOAD_FOLDER, "", 1) finding["sha2"] = utils.gen_sha256_hash( finding["lines"].encode(encoding="utf-8", errors="replace")) security_issues.append(finding) # Vulnerability Multi Regex Match for mulregex in scan_rules["vuln_mul_regex"].iterkeys(): sig_source = scan_rules["vuln_mul_regex"][ mulregex]["sig_source"] sig_line = scan_rules["vuln_mul_regex"][ mulregex]["sig_line"] if re.search(sig_source, data): if re.search(sig_line, line): finding = {} finding["title"] = mulregex finding["description"] = scan_rules[ "desc"][mulregex] finding["tag"] = scan_rules[ "tag"][mulregex] finding["line"] = line_no + 1 finding["lines"] = get_lines( line_no, lines) finding["filename"] = filename finding["path"] = full_file_path.replace( settings.UPLOAD_FOLDER, "", 1) finding["sha2"] = utils.gen_sha256_hash( finding["lines"].encode(encoding="utf-8", errors="replace")) security_issues.append(finding) # Dynamic Regex for dynregex in scan_rules["vuln_dyn_regex"].iterkeys(): signature = scan_rules["vuln_dyn_regex"][ dynregex]["signature"] dyn_sig = scan_rules["vuln_dyn_regex"][ dynregex]["dyn_sig"] sig = re.search(signature, line) if sig: index = line.find(sig.group()) dyn_variable = line[0:index] dyn_variable = dyn_variable.replace( "var", "").replace("=", "").strip() if re.match(r'^[\w]+$', dyn_variable): dyn_sig = dyn_variable + dyn_sig for line_no, nline in enumerate(lines): if re.search(dyn_sig, nline): finding = {} finding["title"] = dynregex finding["description"] = scan_rules[ "desc"][dynregex] finding["tag"] = scan_rules[ "tag"][dynregex] finding[ "line"] = line_no + 1 finding["lines"] = get_lines( line_no, lines) finding[ "filename"] = filename finding[ "path"] = full_file_path.replace(settings.UPLOAD_FOLDER, "", 1) finding["sha2"] = utils.gen_sha256_hash( finding["lines"].encode(encoding="utf-8", errors="replace")) security_issues.append( finding) # Good Finding String Match for good_find in scan_rules["good_to_have_rgx"].iterkeys(): if re.search(scan_rules["good_to_have_rgx"][good_find], line): finding = {} finding["title"] = good_find finding["description"] = scan_rules[ "desc"][good_find] finding["tag"] = scan_rules[ "tag"][good_find] finding["line"] = line_no + 1 finding["lines"] = get_lines( line_no, lines) finding["filename"] = filename finding["path"] = full_file_path.replace( settings.UPLOAD_FOLDER, "", 1) finding["sha2"] = utils.gen_sha256_hash( finding["lines"].encode(encoding="utf-8", errors="replace")) good_finding.append(finding) # Missing Security Headers String Match for header in scan_rules["missing_sec_header"].iterkeys(): if re.search(scan_rules["missing_sec_header"][header], line, re.I): # Good Header is present header_found[header] = 1 # Write Beautifed Data If Any if beautified_data is not None: utils.unicode_safe_file_write( full_file_path, beautified_data) # After Every Files are done. # Check for missing Security Headers for header in header_found.iterkeys(): if header_found[header] == 0: finding = {} finding["title"] = header finding["description"] = scan_rules["desc"][header] finding["tag"] = scan_rules["tag"][header] missing_header.append(finding) #Vulnerability and Count for issue in security_issues: if not issue["title"] in vuln_n_count: vuln_n_count[issue["title"]] = 0 vuln_n_count[issue["title"]] += 1 for issue in security_issues: if not tags[issue["tag"]] in sec_issues_by_tag: sec_issues_by_tag[tags[issue["tag"]]] = [] sec_issues_by_tag[tags[issue["tag"]]].append(issue) for find in good_finding: if not tags[find["tag"]] in good_finding_by_tag: good_finding_by_tag[tags[find["tag"]]] = [] good_finding_by_tag[tags[find["tag"]]].append(find) for sec_header in missing_header: if not tags[sec_header["tag"]] in missing_header_by_tag: missing_header_by_tag[tags[sec_header["tag"]]] = [] missing_header_by_tag[tags[sec_header["tag"]]].append(sec_header) count["sec"] = len(security_issues) count["mis"] = len(missing_header) count["good"] = len(good_finding) scan_results["sec_issues"] = sec_issues_by_tag scan_results["good_finding"] = good_finding_by_tag scan_results["missing_sec_header"] = missing_header_by_tag scan_results["total_count"] = count scan_results["vuln_count"] = vuln_n_count scan_results["files"] = all_files return scan_results except: utils.print_exception("Error Performing Code Analysis")
def ternaryToJson(filename, jsonfile="", outputfilepath="", sheetname="Tie lines", output="ZPF"): """ To convert a spreadsheet file with phase information of ternary system to json file filename: string, the location path of data file in csv or excel format. jsonfile: string, the final desired and saved json file sheetname: string, the sheetname for differetn types of data. It includes "Unary phase", "Tie lines" return: none """ file_extension = os.path.splitext(filename)[1] if file_extension == '.csv': data = pd.read_csv(filename) elif file_extension == '.xls' or file_extension == '.xlsx': if sheetname == "default": sheetname = os.path.splitext(filename)[0] data = pd.read_excel(filename, sheet_name=sheetname) if "Components" in data.columns: components = list(data["Components"].dropna()) else: raise NameError("Components are not in inputfile's columns names") # elements_number = len(components) # refs_content = "" comp1 = components[0] comp2 = components[1] options = jsbeautifier.default_options() options.indent_size = 4 if sheetname == "Tie lines": if "Ref" in data.columns and (not data["Ref"].isnull().values.all()): Refs = data["Ref"].unique() for ref in Refs: df_ref = data[data["Ref"] == ref] Temps = list(df_ref["T/K"]) if "Phase 3" in df_ref.columns: phases = list( np.unique( np.concatenate([ df_ref["Phase 1"].dropna().unique(), df_ref["Phase 2"].dropna().unique(), df_ref["Phase 3"].dropna().unique() ]))) else: phases = list( np.unique( np.concatenate([ df_ref["Phase 1"].dropna().unique(), df_ref["Phase 2"].dropna().unique() ]))) if jsonfile == "": jsonfile_list = components + [output ] + phases + [ref + ".json"] jsonfile_name = createFileName(jsonfile_list) filepath = os.path.join(outputfilepath, jsonfile_name) if not os.path.exists(filepath): # create a json file # os.makedirs(path1) open(filepath, 'w') else: filepath = jsonfile phase1_comp1 = comp1 + "_" + "1" phase1_comp2 = comp2 + "_" + "1" phase2_comp1 = comp1 + "_" + "2" phase2_comp2 = comp2 + "_" + "2" phase3_comp1 = comp1 + "_" + "3" phase3_comp2 = comp2 + "_" + "3" phase_data = [] for i in df_ref.index: phase1 = [ df_ref["Phase 1"][i], [comp1, comp2], [df_ref[phase1_comp1][i], df_ref[phase1_comp2][i]] ] phase2 = [ df_ref["Phase 2"][i], [comp1, comp2], [df_ref[phase2_comp1][i], df_ref[phase2_comp2][i]] ] if not pd.isna(df_ref["Phase 3"][i]): phase3 = [ df_ref["Phase 3"][i], [comp1, comp2], [df_ref[phase3_comp1][i], df_ref[phase3_comp2][i]] ] phase_data.append([phase1, phase2, phase3]) else: phase_data.append([phase1, phase2]) data_dict = { "components": components, "phases": phases, "broadcast_conditions": False, "conditions": { "T": Temps, "P": [101325] }, "output": output, "values": phase_data, "reference": ref, "comment": "" } # json_object = json.dumps(data_dict, indent = 4) json_object = jsbeautifier.beautify(json.dumps(data_dict), options) # write jsonfile with open(filepath, 'w') as outfile: outfile.write(json_object)
def format(self, text): text = text.decode("utf-8") opts = self.formatter.settings.get('codeformatter_js_options') stderr = "" stdout = "" options = jsbeautifier.default_options() if ("indent_size" in self.opts and self.opts["indent_size"]): options.indent_size = self.opts["indent_size"] else: options.indent_size = 4 if ("indent_char" in self.opts and self.opts["indent_char"]): options.indent_char = str(self.opts["indent_char"]) else: options.indent_char = " " if ("indent_with_tabs" in self.opts and self.opts["indent_with_tabs"]): options.indent_with_tabs = True else: options.indent_with_tabs = False if ("eol" in self.opts and self.opts["eol"]): options.eol = self.opts["eol"] else: options.eol = "\n" if ("preserve_newlines" in self.opts and self.opts["preserve_newlines"]): options.preserve_newlines = True else: options.preserve_newlines = False if ("max_preserve_newlines" in self.opts and self.opts["max_preserve_newlines"]): options.max_preserve_newlines = self.opts["max_preserve_newlines"] else: options.max_preserve_newlines = 10 if ("space_in_paren" in self.opts and self.opts["space_in_paren"]): options.space_in_paren = True else: options.space_in_paren = False if ("space_in_empty_paren" in self.opts and self.opts["space_in_empty_paren"]): options.space_in_empty_paren = True else: options.space_in_empty_paren = False if ("e4x" in self.opts and self.opts["e4x"]): options.e4x = True else: options.e4x = False if ("jslint_happy" in self.opts and self.opts["jslint_happy"]): options.jslint_happy = True else: options.jslint_happy = False if ("brace_style" in self.opts and self.opts["brace_style"]): options.brace_style = self.opts["brace_style"] else: options.brace_style = 'collapse' if ("keep_array_indentation" in self.opts and self.opts["keep_array_indentation"]): options.keep_array_indentation = True else: options.keep_array_indentation = False if ("keep_function_indentation" in self.opts and self.opts["keep_function_indentation"]): options.keep_function_indentation = True else: options.keep_function_indentation = False if ("eval_code" in self.opts and self.opts["eval_code"]): options.eval_code = True else: options.eval_code = False if ("unescape_strings" in self.opts and self.opts["unescape_strings"]): options.unescape_strings = True else: options.unescape_strings = False if ("wrap_line_length" in self.opts and self.opts["wrap_line_length"]): options.wrap_line_length = self.opts["wrap_line_length"] else: options.wrap_line_length = 0 if ("break_chained_methods" in self.opts and self.opts["break_chained_methods"]): options.break_chained_methods = True else: options.break_chained_methods = False if ("end_with_newline" in self.opts and self.opts["end_with_newline"]): options.end_with_newline = True else: options.end_with_newline = False if ("comma_first" in self.opts and self.opts["comma_first"]): options.comma_first = True else: options.comma_first = False if ("space_after_anon_function" in self.opts and self.opts["space_after_anon_function"]): options.space_after_anon_function = True else: options.space_after_anon_function = False try: stdout = jsbeautifier.beautify(text, options) except Exception as e: stderr = str(e) #return "", "" if (not stderr and not stdout): stderr = "Formatting error!" return stdout, stderr
def save(self, *args, **kwargs): self.service_js = jsbeautifier.beautify(self.service_js) super(DjangularService, self).save(*args, **kwargs)
def save(self, *args, **kwargs): self.controller_js = jsbeautifier.beautify(self.controller_js) super(DjangularSlaveViewController, self).save(*args, **kwargs)
def run(self, text): import jsbeautifier return jsbeautifier.beautify(text)
# TODO wrap this up nicely scraper = PokeScrapr() output = [] with open('pokemon.txt') as f: for line in f.readlines()[:]: pokemon = line.strip().capitalize() print(pokemon) try: js = scraper.get_FSP_JSON(pokemon) print(js) output.append(js) time.sleep(3) except: print("error - ", pokemon, ". sleeping for 5 sec") time.sleep(5) scraper = PokeScrapr() js = scraper.get_FSP_JSON(pokemon) print(js) output.append(js) with open('pokemon-js.js', 'w') as f: json = jsbeautifier.beautify(','.join(output)) f.write(json)
def analyseJS(code, context=None, manualAnalysis=False): ''' Hooks the eval function and search for obfuscated elements in the Javascript code @param code: The Javascript code (string) @return: List with analysis information of the Javascript code: [JSCode,unescapedBytes,urlsFound,errors,context], where JSCode is a list with the several stages Javascript code, unescapedBytes is a list with the parameters of unescape functions, urlsFound is a list with the URLs found in the unescaped bytes, errors is a list of errors, context is the context of execution of the Javascript code. ''' errors = [] JSCode = [] unescapedBytes = [] urlsFound = [] try: code = unescapeHTMLEntities(code) scriptElements = re.findall(reJSscript, code, re.DOTALL | re.IGNORECASE) if scriptElements != []: code = '' for scriptElement in scriptElements: code += scriptElement + '\n\n' code = jsbeautifier.beautify(code) JSCode.append(code) if code != None and JS_MODULE and not manualAnalysis: if context == None: context = PyV8.JSContext(Global()) context.enter() # Hooking the eval function context.eval('eval=evalOverride') #context.eval(preDefinedCode) while True: originalCode = code try: context.eval(code) evalCode = context.eval('evalCode') evalCode = jsbeautifier.beautify(evalCode) if evalCode != '' and evalCode != code: code = evalCode JSCode.append(code) else: break except: error = str(sys.exc_info()[1]) open('jserror.log', 'ab').write(error + newLine) errors.append(error) break if False: escapedVars = re.findall('(\w*?)\s*?=\s*?(unescape\((.*?)\))', code, re.DOTALL) for var in escapedVars: bytes = var[2] if bytes.find('+') != -1 or bytes.find('%') == -1: varContent = getVarContent(code, bytes) if len(varContent) > 150: ret = unescape(varContent) if ret[0] != -1: bytes = ret[1] urls = re.findall('https?://.*$', bytes, re.DOTALL) if bytes not in unescapedBytes: unescapedBytes.append(bytes) for url in urls: if url not in urlsFound: urlsFound.append(url) else: bytes = bytes[1:-1] if len(bytes) > 150: ret = unescape(bytes) if ret[0] != -1: bytes = ret[1] urls = re.findall('https?://.*$', bytes, re.DOTALL) if bytes not in unescapedBytes: unescapedBytes.append(bytes) for url in urls: if url not in urlsFound: urlsFound.append(url) except: traceback.print_exc(file=open(errorsFile, 'a')) errors.append('Unexpected error in the JSAnalysis module!!') finally: for js in JSCode: if js == None or js == '': JSCode.remove(js) return [JSCode, unescapedBytes, urlsFound, errors, context]
import jsbeautifier # sample_product = 'https://www.unisport.dk/api/products/batch/?list=189214' products_men = 'https://www.unisport.dk/api/products/batch/?list=189214,188894,188896,187812,187743,174635,189187,188534,193539,189413,189188,188532,194477,187378,187404,193592,192837,193593,189052,189213,187361,193537,187738,187744,187424,187746,185074,184775,191599,187416,189212,186121,187814,193459,187813,181393,184771,187793,187717,181372,187745,181382,187425,181272,188914,193455,187431,188893,187373,187845,189032,187795,187718,187398,189216,187719,187417,188895,188571,189211,188940,193480,187739,188897,187741,187947,187349,188903,187720,187815,188918,187749,187796,188900,185078,187391,188735,188919,188924,184774,189057,181361,188569,187411,181332,187721,187354,187844,188961,188933,189215,187794,187945,188916,193466,181289,189056,181407,188907,187740,187341,188959,187377,188932,187810,187366,193468,187946,187405,187724,188819,179845,193600,187722,188970,187723,187798,188948,107,186046,188909,193573,187809,188926,188908,193446,188965,188930,187978,186043,189047,187452,188904,187750,180206,189054,187375,193465,188530,187799,187830,188734,188902,188928,188535,187453,187785,179854,187797,181378,189034,188921,187433,188832,187811,193536,192799,188574,181295,188901,187426,184744,193541,186045,188528,179351,193577,181265,193574,187933,184737,187808,141313,188934,189217,188527,185069,193599,189045,188548,187932,187470,188915,188823,179841,187367,186054,188830,185072,193575,187756,184769,188905,181399,193443,181271,184847,187800,188581,187342' producs_kids = 'https://www.unisport.dk/api/products/batch/?list=193974,193976,187411,187452,187412,187426,193595,187427,187434,187457,187428,193596,187413,193594,187447,187437,193589,193587,187430,187414,187449,193585,187439,193588,187429,187415,193586,187448,187441,195867,195869,195877,195873,193541,193540,193542,193479,193473,193476,193469,193477,193472,193474,193471,193475,193470,193478,189217,189033,187341,186087,187933,187830,186092,189034,186090,186076,189035,189037,187777,187772,187774,189056,187847,187453,187939,187470,187755,187473,187400,189046,187776,187941,187947,187845,186116,187749,186118,186117,189047,187376,187454,187471,187843,187944,187751,189048,187474,187773,192056,189059,192052,192508,192344,189060,189061,189062,188554,188930,186089,188926,188931,188927,188835,188932,188933,188836,188934,188928,188552,188935,188929,188936,188553,188937,188837,188950,188969,188951,188972,188952,186075,188948,188970,188973,188971,188949,188954,188974,188925,187851,187932,187936,187849,188828,187832,187771,187935,187937,187753,187945,187942,186086,186072,186103,186091,186088,186115,188587,188596,188598,188588,188599,188589,188597,187852,185079,185081,185080,184777,187850,187931,187342,187848,187841,187934,184780,187831,187770,187840,184778,187846,187938,184779,187754,184781,187775,187837,187940,187747,184760,187844,187946,187750,184758,187838,187842,184755,184757,187943,187752,184759,180230,180187,180190,184714,179351,179355,179457,179453' products_women = 'https://www.unisport.dk/api/products/batch/?list=193957,193974,193966,193969,193963,193960,193976,196165,196161,186096,186048,187404,187416,193592,187411,187424,187452,187431,193574,193575,187418,187432,187405,186121,187417,187412,187426,193854,193593,193595,187425,193852,187427,187433,187434,187468,193573,187829,187406,187423,193855,193572,193853,187435,193576,187420,187407,193579,187443,187442,193571,193581,187456,187408,193591,187419,187457,187428,187413,193590,187469,193594,187447,187436,193584,193589,187475,193582,193583,187476,187421,187430,187422,193578,187410,193588,187429,187415,187445,193570,193586,187448,187440,187441,195875,195863,195867,193537,193541,187361,193465,193466,193539,182529,193443,189601,193455,193468,193462,193536,193540,193538,187354,193542,193453,193479,193467,193464,193447,193469,193461,193477,193450,193472,193454,193474,193448,193470,193460,193478,188530,189213,189217,189211,187812,189212,189033,187743,187341,189626,187793,189625,186046,187810,186087,187933,186043,189214,187738,186095,187830,189636,189638,194239,186068,186061,186071,189216,186064,186092,189034,186090,186051,186076,189215,187762,187378,189035,189037,187797,187781,187777,187772,187380,189036,187788,187774,189629,189052,189605,189056,182521,189604,182525,189057,182523,182527,187807,187709,187458,187768,187472,187780,187355,187401,187367,187802,187712,187966,187847,187453,187939,187470,189054,187765,187399,189044,187377,187755,187473' urls = {} # urls['sample_product'] = sample_product urls['products_men'] = products_men urls['producs_kids'] = producs_kids urls['products_women'] = products_women for name, url in urls.items(): print('Getting {products}'.format(products=name)) response = requests.get(url) # converting response to json object data = json.dumps(response.json()) data = jsbeautifier.beautify(data) # storing json data filename = 'www/webshop/{name}.json'.format(name=name) print('Saving {products} to {filename}'.format(products=name, filename=filename)) with open(filename, "w+") as file: file.writelines(data) print('..done!\n')
import requests import jsbeautifier import json headers = { "User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Ubuntu Chromium/48.0.2564.82 Chrome/48.0.2564.82 Safari/537.36" } """ from 0 to 10 and when using 10 date is 2015-11-17 So for last 6 months issues use upto 10 """ base_url = 'http://www.ichangemycity.com/card/get_complaints/2?agency=0&category_id=0&lat=12.9715987&lon=77.59456269999998&status_id=0&sub_category=67' html = requests.get(base_url, headers=headers) htmltext = html.text res = jsbeautifier.beautify(htmltext) data = json.loads(htmltext) print data['status'] length_result = len(data['result']) print length_result for i in range(0, length_result): print data['result'][i]['complaint_created'] print data['result'][i]['complaint_title'] print data['result'][i]['complaint_description'] print data['result'][i]['complaint_location'] print data['result'][i]['complaint_address_1'] print data['result'][i]['category_name'] print data['result'][i]['ward_name'] print data['result'][i]['complaint_source_id'] print data['result'][i]['civic_agency_name'] #print res
def save_file(directory=os.path.join(os.getcwd(),'jutility.js')): f = open(directory, 'w+') #f.write(str(s)) f.write(jsbeautifier.beautify(s)) f.close()
async def pre_ast(self): while not self.target_queue.empty(): fileext = self.target_queue.get() if not self.lan: break if fileext[0] in ext_dict['php'] and 'php' in self.lan: # 下面是对于php文件的处理逻辑 for filepath in fileext[1]['list']: all_nodes = [] filepath = self.get_path(filepath) self.pre_result[filepath] = {} self.pre_result[filepath]['language'] = 'php' self.pre_result[filepath]['ast_nodes'] = [] fi = codecs.open(filepath, "r", encoding='utf-8', errors='ignore') code_content = fi.read() fi.close() # self.pre_result[filepath]['content'] = code_content try: parser = make_parser() all_nodes = parser.parse(code_content, debug=False, lexer=lexer.clone(), tracking=True) # 合并字典 self.pre_result[filepath]['ast_nodes'] = all_nodes except SyntaxError as e: logger.warning('[AST] [ERROR] parser {}: {}'.format(filepath, traceback.format_exc())) except AssertionError as e: logger.warning('[AST] [ERROR] parser {}: {}'.format(filepath, traceback.format_exc())) except: logger.warning('[AST] something error, {}'.format(traceback.format_exc())) # 搜索所有的常量 for node in all_nodes: if isinstance(node, php.FunctionCall) and node.name == "define": define_params = node.params logger.debug( "[AST][Pretreatment] new define {}={}".format(define_params[0].node, define_params[1].node)) self.define_dict[define_params[0].node] = define_params[1].node elif fileext[0] in ext_dict['chromeext'] and 'chromeext' in self.lan: # 针对chrome 拓展的预处理 # 需要提取其中的js和html? for filepath in fileext[1]['list']: child_files = [] child_files_html = [] filepath = self.get_path(filepath) self.pre_result[filepath] = {} self.pre_result[filepath]['language'] = 'chromeext' # 首先想办法解压crx try: target_files_path = un_zip(filepath) self.pre_result[filepath]['target_files_path'] = target_files_path except zipfile.BadZipFile: logger.warning("[Pretreatment][Chrome Ext] file {} not zip".format(filepath)) continue except OSError: logger.warning("[Pretreatment][Chrome Ext] file {} unzip error".format(filepath)) continue # 分析manifest.json manifest_path = os.path.join(target_files_path, "manifest.json") # target可能是单个文件,这里需要专门处理 if not (self.target_directory.endswith("/") or self.target_directory.endswith("\\")) and not os.path.isdir(self.target_directory): path_list = re.split(r'[\\|/]', self.target_directory) relative_path = os.path.join(path_list[-1]+"_files") else: relative_path = target_files_path.split(self.target_directory)[-1] if relative_path.startswith('\\') or relative_path.startswith("/"): relative_path = relative_path[1:] if os.path.isfile(manifest_path): fi = codecs.open(manifest_path, "r", encoding='utf-8', errors='ignore') manifest_content = fi.read() fi.close() try: manifest = json.loads(manifest_content, encoding='utf-8') except json.decoder.JSONDecodeError: logger.warning( "[Pretreatment][Chrome Ext] File {} parse error...".format(target_files_path)) continue self.pre_result[filepath]["manifest"] = manifest # 想办法优化,如果不想深入js和html的判断,那么就跳过 if len(self.lan) and self.lan == 'chromeext': logger.debug("[Pretreatment][Chrome Ext] pass js & html scan...") continue # content scripts if "content_scripts" in manifest: for script in manifest["content_scripts"]: if "js" in script: child_files.extend([os.path.join(relative_path, js) for js in script['js']]) # background js if "background" in manifest: if "scripts" in manifest["background"]: child_files.extend([os.path.join(relative_path, js) for js in manifest["background"]["scripts"]]) # background html if "page" in manifest["background"]: child_files_html.append(os.path.join(relative_path, manifest["background"]["page"])) # popup.html if "browser_action" in manifest: if "default_popup" in manifest["browser_action"]: child_files_html.append(os.path.join(relative_path, manifest["browser_action"]["default_popup"])) # web_accessible_resources if "web_accessible_resources" in manifest: for resource in manifest["web_accessible_resources"]: if ".js" in resource: child_files.append(os.path.join(relative_path, resource)) if ".html" in resource: child_files_html.append(os.path.join(relative_path, resource)) # chrome_url_overrides if "chrome_url_overrides" in manifest: for key in manifest["chrome_url_overrides"]: child_files_html.append(os.path.join(relative_path, manifest["chrome_url_overrides"][key])) self.pre_result[filepath]["child_files"] = child_files if len(child_files): # 将content_scripts加入到文件列表中构造 self.target_queue.put(('.js', {'count': len(child_files), 'list': child_files})) # 通过浅复制操作外部传入的files self.file_list.append(('.js', {'count': len(child_files), 'list': child_files})) if len(child_files_html): self.target_queue.put(('.html', {'count': len(child_files_html), 'list': child_files_html})) else: logger.warning("[Pretreatment][Chrome Ext] File {} parse error...".format(target_files_path)) continue elif fileext[0] in ext_dict['html'] and 'javascript' in self.lan: # html only found js for filepath in fileext[1]['list']: filepath = self.get_path(filepath) script_list = [] try: fi = codecs.open(filepath, "r", encoding='utf-8', errors='ignore') code_content = fi.read() fi.close() except FileNotFoundError: continue except OSError: continue # tmp.js save all inline javascript code tmp_path = os.path.join(os.path.dirname(filepath), "tmp.js") fi2 = codecs.open(tmp_path, "a", encoding='utf-8', errors='ignore') try: soup = BeautifulSoup(code_content, "html.parser") script_tag_list = soup.find_all('script') for script_tag in script_tag_list: script_attrs = script_tag.attrs if 'src' in script_attrs: parents_path = os.path.normpath("\\".join(re.split(r'[\\|/]', filepath)[:-1])) script_path = os.path.join(parents_path, script_attrs['src']) script_list.append(script_path) else: # 如果没有src,那么代表是内联js script_content = script_tag.string fi2.write(" \n{}\n ".format(script_content)) fi2.close() if tmp_path not in script_list: script_list.append(tmp_path) # 将content_scripts加入到文件列表中构造 self.target_queue.put(('.js', {'count': len(script_list), 'list': script_list})) # 通过浅复制操作外部传入的files self.file_list.append(('.js', {'count': len(script_list), 'list': script_list})) except: logger.warning('[AST] something error, {}'.format(traceback.format_exc())) continue elif fileext[0] in ext_dict['javascript'] and 'javascript' in self.lan: # 针对javascript的预处理 # 需要对js做语义分析 for filepath in fileext[1]['list']: filepath = self.get_path(filepath) if not filepath.endswith(".js"): continue self.pre_result[filepath] = {} self.pre_result[filepath]['language'] = 'javascript' self.pre_result[filepath]['ast_nodes'] = [] try: fi = codecs.open(filepath, "r", encoding='utf-8', errors='ignore') code_content = fi.read() fi.close() except FileNotFoundError: continue except OSError: continue # 添加代码美化并且写入新文件 new_filepath = filepath + ".pretty" try: if not os.path.isfile(new_filepath): fi2 = codecs.open(new_filepath, "w", encoding='utf-8', errors='ignore') code_content = jsbeautifier.beautify(code_content) fi2.write(code_content) fi2.close() # self.pre_result[filepath]['content'] = code_content all_nodes = esprima.parse(code_content, {"loc": True}) # 合并字典 self.pre_result[filepath]['ast_nodes'] = all_nodes except SyntaxError as e: logger.warning('[AST] [ERROR] parser {}: {}'.format(filepath, traceback.format_exc())) except AssertionError as e: logger.warning('[AST] [ERROR] parser {}: {}'.format(filepath, traceback.format_exc())) except esprima.error_handler.Error: logger.warning('[AST] [ERROR] Invalid regular expression in {}...'.format(filepath)) except KeyboardInterrupt: logger.log('[AST stop...') exit() except: logger.warning('[AST] something error, {}'.format(traceback.format_exc())) continue # 手动回收? gc.collect() return True
have_stop = True if not have_stop and c in cpnet_vocab: prune_qc.append(c) ac = item["ac"] prune_ac = [] for c in ac: if c[-2:] == "er" and c[:-2] in ac: continue if c[-1:] == "e" and c[:-1] in ac: continue all_stop = True for t in c.split("_"): if t not in nltk_stopwords: all_stop = False if not all_stop and c in cpnet_vocab: prune_ac.append(c) item["qc"] = prune_qc item["ac"] = prune_ac prune_data.append(item) import jsbeautifier opts = jsbeautifier.default_options() opts.indent_size = 2 with open(sys.argv[1], 'w') as fp: # json.dump(prune_data, f) fp.write(jsbeautifier.beautify(json.dumps(prune_data), opts))
def decodesto(self, input, expectation=None): self.assertEqual(jsbeautifier.beautify(input, self.options), expectation or input)
import jsbeautifier import clipboard import pyautogui target = clipboard.paste() opts = jsbeautifier.default_options() opts.type = "html" res = jsbeautifier.beautify(target, opts) clipboard.copy(res)
from framescript.Lexer import Lexer from framescript.Parser import Parser from framescript.Transpiler import Transpiler import sys import jsbeautifier if __name__ == '__main__': lexer = Lexer(open(sys.argv[1]).read()) parser = Parser(lexer) transpiler = Transpiler() tree = parser.parse() x = transpiler.visit(tree) y = transpiler.finalize() print(y) print('<script>') print(jsbeautifier.beautify(x)) print('</script>')
def save(self, *args, **kwargs): self.directive_js = jsbeautifier.beautify(self.directive_js) super(DjangularDirective, self).save(*args, **kwargs)
def beautifyJS(self, content): return jsbeautifier.beautify(content)
def save(self, *args, **kwargs): self.module_js = jsbeautifier.beautify(self.module_js) self.controller_js = jsbeautifier.beautify(self.controller_js) super(DjangularMasterViewController, self).save(*args, **kwargs)
def beautifier_test_underscore(): jsbeautifier.beautify(data, options)
def pret(code): res = jsbeautifier.beautify(code) return res
def beautifier_test_underscore_min(): jsbeautifier.beautify(data_min, options)
def generate(self, beautify=True): code = self.template + \ super(JSGenerator, self).generate(self.get_menu_func) if beautify: code = jsbeautifier.beautify(code) return code
baseUrl = 'http://127.0.0.1:2330/project/public' log.info( " ================================ 收藏测试 ================================ ") print() stopwatch = Stopwatch() stopwatch.start() try: res = requests.post(baseUrl + "/auth/nonce", data={}, allow_redirects=False) jsonRaw = jsbeautifier.beautify(res.text) #log.response(jsonRaw, 1) jsonObj = json.loads(jsonRaw) email = '*****@*****.**' password = '******' nonce = jsonObj['data']['nonce'] sign = sha1(nonce + email + sha1("moeje" + password)) log.info("sign: " + sign) res = requests.post(baseUrl + "/auth/login/email", data={ 'email': email, 'nonce': nonce, 'sign': sign }, allow_redirects=False) jsonRaw = jsbeautifier.beautify(res.text)
def binaryToJson(filename, jsonfile="", outputfilepath="", sheetname="default", output="ZPF"): """ filename: string path: string return: none """ file_extension = os.path.splitext(filename)[1] if file_extension == '.csv': data = pd.read_csv(filename) elif file_extension == '.xls' or file_extension == '.xlsx': if sheetname == "default": sheetname = os.path.splitext(filename)[0] data = pd.read_excel(filename, sheet_name=sheetname) if "Components" in data.columns: components = list(data["Components"].dropna()) else: raise NameError("Components are not in inputfile's columns names") options = jsbeautifier.default_options() options.indent_size = 4 if sheetname == "Tie lines": if "Ref" in data.columns and (not data["Ref"].isnull().values.all()): Refs = data["Ref"].unique() for ref in Refs: df_ref = data[data["Ref"] == ref] Temps = list(df_ref["T/K"]) if "Phase 3" in df_ref.columns: phases = list( np.unique( np.concatenate([ df_ref["Phase 1"].dropna().unique(), df_ref["Phase 2"].dropna().unique(), df_ref["Phase 3"].dropna().unique() ]))) else: phases = list( np.unique( np.concatenate([ df_ref["Phase 1"].dropna().unique(), df_ref["Phase 2"].dropna().unique() ]))) if jsonfile == "": jsonfile_list = components + [output ] + phases + [ref + ".json"] jsonfile_name = createFileName(jsonfile_list) filepath = os.path.join(outputfilepath, jsonfile_name) if not os.path.exists(filepath): # create a json file # os.makedirs(path1) open(filepath, 'w') else: filepath = jsonfile comp1 = components[0] phase1_comp1 = comp1 + "_" + "1" phase2_comp1 = comp1 + "_" + "2" phase_data = [] for i in df_ref.index: phase1 = [ df_ref["Phase 1"][i], [comp1], [df_ref[phase1_comp1][i]] ] phase2 = [ df_ref["Phase 2"][i], [comp1], [df_ref[phase2_comp1][i]] ] phase_data.append([phase1, phase2]) data_dict = { "components": components, "phases": phases, "broadcast_conditions": False, "conditions": { "T": Temps, "P": [101325] }, "output": output, "values": phase_data, "reference": ref, "comment": "" } json_object = jsbeautifier.beautify(json.dumps(data_dict), options) with open(filepath, 'w') as outfile: outfile.write(json_object)
import os import re from lxml import html import requests import jsbeautifier response = requests.get('http://app.youneedabudget.com') with open('index.html', 'w', encoding='utf-8') as file_before: file_before.write(response.text) parsed = html.fromstring(response.text) for src in parsed.xpath('//script/@src'): url_src = str(src) file = url_src.rsplit('/', 1)[-1] if file.startswith('before.'): before_response = requests.get(str(src)) before_script = jsbeautifier.beautify(before_response.text) with open(os.path.join('web_app', 'before.js'), 'w+', encoding='utf-8') as file_before: file_before.write(before_script) regex1 = re.compile('\s*(\d)\:\s\"appmain\"') regex2 = None for line in before_script.split('\n'): if regex1.match(line): idx = regex1.match(line).groups()[0] regex2 = re.compile('\s*%s\:\s\"(.*)\"' % idx) if regex2 is not None and regex2.match(line): test = regex2.match(line).groups()[0] if test != 'appmain': random_id = test break url_appmain = '/'.join(
def beautifyjs(content): try: return jsbeautifier.beautify(content) except: pass
def build_page(self, path): opts = jsbeautifier.default_options() opts.indent_size = 2 opts.max_preserve_newlines = 30 with open(f"{path}/dashboard-routing.module.ts", 'w') as ts_file: ts_file.write(jsbeautifier.beautify(self.build_ts(), opts))
def __call__(self, data, **metadata): opts = jsbeautifier.default_options() opts.indent_size = 2 data = data.decode("utf-8", "replace") res = jsbeautifier.beautify(data, opts) return "JavaScript", base.format_text(res)
def format_selection(self, edit, opts): def get_line_indentation_pos(view, point): line_region = view.line(point) pos = line_region.a end = line_region.b while pos < end: ch = view.substr(pos) if ch != ' ' and ch != '\t': break pos += 1 return pos def get_indentation_count(view, start): indent_count = 0 i = start - 1 while i > 0: ch = view.substr(i) scope = view.scope_name(i) # Skip preprocessors, strings, characaters and comments if 'string.quoted' in scope or 'comment' in scope or 'preprocessor' in scope: extent = view.extract_scope(i) i = extent.a - 1 continue else: i -= 1 if ch == '}': indent_count -= 1 elif ch == '{': indent_count += 1 return indent_count view = self.view regions = [] for sel in view.sel(): start = get_line_indentation_pos(view, min(sel.a, sel.b)) region = sublime.Region( view.line(start).a, # line start of first line view.line(max(sel.a, sel.b)).b) # line end of last line indent_count = get_indentation_count(view, start) # Add braces for indentation hack code = '{' * indent_count if indent_count > 0: code += '\n' code += view.substr(region) # Performing astyle formatter formatted_code = jsbeautifier.beautify(code, opts) if indent_count > 0: for _ in range(indent_count): index = formatted_code.find('{') + 1 formatted_code = formatted_code[index:] formatted_code = re.sub(r'[ \t]*\n([^\r\n])', r'\1', formatted_code, 1) else: # HACK: While no identation, a '{' will generate a blank line, so strip it. search = "\n{" if search not in code: formatted_code = formatted_code.replace(search, '{', 1) # Applying formatted code view.replace(edit, region, formatted_code) # Region for replaced code if sel.a <= sel.b: regions.append(sublime.Region(region.a, region.a + len(formatted_code))) else: regions.append(sublime.Region(region.a + len(formatted_code), region.a)) view.sel().clear() # Add regions of formatted code [view.sel().add(region) for region in regions]