def _ReadDocumentConfig(documentConfigFilePath: str) -> typing.Tuple[str, str]: """ :return: Document build path, Document text, Indexing information """ with open(documentConfigFilePath) as documentConfigFile: documentConfig = decoder.JSONDecoder().decode( documentConfigFile.read()) # type: dict documentTemplateConfigFilePath = os.path.join( Paths.DocumentsConfigTemplatesPath, documentConfig["TemplateConfig"]) # type: str with open(documentTemplateConfigFilePath) as documentTemplateConfigFile: documentTemplateConfig = decoder.JSONDecoder().decode( documentTemplateConfigFile.read()) # type: dict documentFileExtension = documentConfig["FileExtension"] # type: str documentRelativeFilePath = os.path.splitext( documentConfigFilePath.replace( Paths.DocumentsConfigDocumentsPath + os.path.sep, ""))[0] + "." + documentFileExtension # type: str documentRelativeFilePath.replace(os.path.altsep, os.path.sep) documentFilePath = os.path.join(Paths.DocumentsBuildPath, documentRelativeFilePath) documentAdditionalValues = {"DocumentPath": documentRelativeFilePath} documentValues = _ReadValues(documentConfig, documentAdditionalValues) # type: dict documentValues["DocumentPath"] = documentRelativeFilePath if documentValues.get("BasePath") is None: documentRelativeFilePathParts = documentRelativeFilePath.split( os.path.sep) # type: typing.List[str] documentBasePath = "../" * (len(documentRelativeFilePathParts) - 1) documentValues["BasePath"] = documentBasePath documentText = _ReadDocument( documentValues, documentTemplateConfig["Document"]) # type: str return documentFilePath, documentText
def __request_post(self): ''' http://d.web2.qq.com/channel/login2 r:{"status":"online","ptwebqq":"95b148b95af9be7677757b3a629e3904f52f153d0b714c527f81f8d9e385867a","passwd_sig":"", "clientid":"21628014","psessionid":null} clientid:21628014 psessionid:null ''' self.__headers.update({ 'Referer': 'http://d.web2.qq.com/proxy.html?v=20110331002&callback=2' }) a = { 'status': 'online', 'ptwebqq': self.__getcookies('ptwebqq'), 'passwd_sig': '', 'clientid': self.__clientid, 'psessionid': 'null' } array = { 'r': json_encode.JSONEncoder().encode(a), 'clientid': self.__clientid, 'psessionid': 'null' } url = 'http://d.web2.qq.com/channel/login2' str = self.__request(url, 'POST', array) str = json_decode.JSONDecoder().decode(str) self.__psessionid = str['result']['psessionid'] self.__vfwebqq = str['result']['vfwebqq'] self.__get_friend_info2() self.__get_user_friends2() self.__get_group_name_list_mask2() self.__poll2_() self.__get_msg_tip_() pass
def _Setup() -> None: global _mod, _modData informationFilePath = os.path.join( os.path.dirname(os.path.dirname(os.path.normpath(__file__))), "Mod.json") # type: str try: with open(os.path.join(informationFilePath)) as informationFile: _mod = Mod(decoder.JSONDecoder().decode(informationFile.read())) except Exception as e: raise Exception("Failed to read mod information for '" + informationFilePath + "'. \n") from e _modData = { "Namespace": GetCurrentMod().Namespace, "Name": GetCurrentMod().Name, "Version": GetCurrentMod().Version, "ChangesFilePath": GetCurrentMod().ChangesFilePath, "PlansFilePath": GetCurrentMod().PlansFilePath, "InstallerFilePath": GetCurrentMod().DistributionInstallerFilePath, "FilesFilePath": GetCurrentMod().DistributionFilesFilePath, "SourcesFileName": GetCurrentMod().SourcesFileName, "GithubName": GetCurrentMod().GithubName } # type: typing.Dict[str, typing.Any]
def scrape_shortcodes(self, initial_url, ): """ Scrapes shortcodes contained in initial_url (ie. the redirect information for urls of the page's posts) Input: initial_url : url to initial page Output: df: pandas DataFrame with relevant information from scraped posts """ is_more = True shortcodes = [] next_url = initial_url while is_more: response = requests.get(next_url) print(response.status_code) soup = BeautifulSoup(response.text, "html.parser") decode = decoder.JSONDecoder() home_page = decode.decode(str(soup)) for i in range(len(home_page['data']['user']['edge_owner_to_timeline_media']['edges'])): sc = home_page['data']['user']['edge_owner_to_timeline_media']['edges'][i]['node']['shortcode'] shortcodes.append(sc) time.sleep(self.sleep) if home_page['data']['user']['edge_owner_to_timeline_media']['page_info']['has_next_page'] == True: nextpage = home_page['data']['user']['edge_owner_to_timeline_media']['page_info']['end_cursor'] next_url = 'https://www.instagram.com/graphql/query/?query_id=17888483320059182&variables=%7B%22id%22%3A%222183125078%22%2C%22first%22%3A12%2C%22after%22%3A%22{}%22%7D'.format(nextpage) else: is_more = False self.shortcodes = shortcodes
def extended_json_decoder(): '''Return a decoder for the superset of JSON understood by the upatch function. The exact nature of this superset is documented in the manpage for json_patch(1). Briefly, the string ``...`` is accepted where standard JSON expects a ``{<property name>}: {<object>}`` construction, and the string ``...``, optionally followed by a number in parentheses, is accepted where standard JSON expects an array element. The superset is implemented by parsing ``...`` in JSON objects as a key/value pair ``Ellipsis: True`` in the resulting dictionary, and ``...({num}){?}`` as a subsequence of ``{num}`` ``Ellipsis`` objects, or one ellipsis object if ``({num})`` is not present. Examples: >>> dec = extended_json_decoder() >>> (dec.decode('{"foo": "bar",...,"baz": "quux"}') == ... {"foo": "bar", "baz": "quux", Ellipsis: True}) True >>> dec.decode('[...]') [Ellipsis] >>> dec.decode('["foo",...(3),"bar"]') [u'foo', Ellipsis, Ellipsis, Ellipsis, u'bar'] ''' dec = decoder.JSONDecoder() dec.parse_object = _JSONObject dec.parse_array = _JSONArray dec.scan_once = scanner.py_make_scanner(dec) return dec
def _SetupExternalPaths() -> None: global DistributionReleasesPath, DistributionPreviewsPath, WebsitesDistributionPath with open(os.path.join(SettingsPath, "ExternalPaths.json")) as externalPathsFile: externalPathsInformation = decoder.JSONDecoder().decode( externalPathsFile.read()) distributionReleasesRelativePath = externalPathsInformation.get( "DistributionReleasesPath", None) # type: str if distributionReleasesRelativePath is not None: DistributionReleasesPath = os.path.normpath( os.path.abspath( os.path.join(AutomationPath, distributionReleasesRelativePath))) distributionPreviewsRelativePath = externalPathsInformation.get( "DistributionPreviewsPath", None) # type: str if distributionPreviewsRelativePath is not None: DistributionPreviewsPath = os.path.normpath( os.path.abspath( os.path.join(AutomationPath, distributionPreviewsRelativePath))) websitesDistributionRelativePath = externalPathsInformation.get( "WebsitesDistributionPath", None) # type: str if websitesDistributionRelativePath is not None: WebsitesDistributionPath = os.path.normpath( os.path.abspath( os.path.join(AutomationPath, websitesDistributionRelativePath)))
def test_json_object(): pairs, end = parse_json_object(('"abc":213}', 0), 'utf-8') print pairs, type(pairs) jd = decoder.JSONDecoder() pairs, end = decoder.JSONObject(('"abc":213}', 0), 'utf-8', scan_once = jd.scan_once, strict=False, object_hook=None, object_pairs_hook=None) print pairs, type(pairs) p = [] print type(p) p.append(('abc', 213)) print type(p)
def __init__(self): self.__socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) with open("config.conf", 'r') as content_file: decoder = decoder_json.JSONDecoder() decoded = decoder.decode(content_file.read()) host = decoded['host'] port = int(decoded['port']) server_address = (host, port) self.__socket.bind(server_address) # listen() puts the socket into server mode self.__socket.listen(1000) from Server.controllers.RoomsController import RoomsController self.__room_controller_thread = RoomsController()
def read_config() -> dict: if os.path.exists("config.json"): cfg_decoder = decoder.JSONDecoder() with open("config.json", 'rt') as f: return cfg_decoder.decode('\n'.join(f.readlines())) else: print('Config file not found. Creating new one...') cfg = create_default_config() cfg_encoder = encoder.JSONEncoder() json = cfg_encoder.encode(cfg) with open("config.json", 'wt') as f: f.write(json) f.flush() return cfg
def GetModVersion(self) -> str: with open(self.InformationSourceFilePath) as informationFile: informationDictionary = decoder.JSONDecoder().decode( informationFile.read()) # type: typing.Dict[str, typing.Any] if not "Version" in informationDictionary: raise ValueError("Entry 'Version' does not exist.") version = informationDictionary["Version"] if not isinstance(version, str): raise TypeError("Entry 'Version' is not a string.") return version
def parse_rate(self, response): item = SupItem() item['website'] = response.meta['link'] item['word'] = response.meta['word'] json = response.body[response.body.find('{'):-2] print 'json->>>>>>>>>>>>>>>>>>>>>>>', json jsonD = decoder.JSONDecoder() djson = jsonD.decode(json) item['lastOneWeek'] = djson['data']['starStatisticsList'][5]['weekly'] item['lastOneMonth'] = djson['data']['starStatisticsList'][5][ 'monthly'] item['lastSixMonth'] = djson['data']['starStatisticsList'][5][ 'halfYearly'] item['beforeHalfYear'] = djson['data']['starStatisticsList'][5][ 'halfYearBefore'] item['total'] = djson['data']['starStatisticsList'][5]['totals'] return item
def __call__(self, *args, **kwargs): postdata = json_encoder_class().encode({ 'method': self.__serviceName, 'params': args + (kwargs, ), 'id': 'jsonrpc' }) request = urllib2.Request(self.__serviceURL, data=postdata, headers=self.__headers) respdata = urllib2.urlopen(request).read() try: resp = decoder.JSONDecoder().decode(respdata) except ValueError: raise JSONRPCException('Error decoding JSON reponse:\n' + respdata) if resp['error'] is not None: raise BuildException(resp['error']) else: return resp['result']
def iter_response_objects(response): prev = '' decoder = json_decoder.JSONDecoder() for chunk in response.read_chunked(decode_content=False): if isinstance(chunk, bytes): chunk = chunk.decode('utf-8') chunk = prev + chunk length = len(chunk) idx = json_decoder.WHITESPACE.match(chunk, 0).end() while idx < length: try: message, idx = decoder.raw_decode(chunk, idx) except ValueError: # malformed or incomplete JSON, unlikely to happen break else: yield message idx = json_decoder.WHITESPACE.match(chunk, idx).end() prev = chunk[idx:]
def _Setup () -> None: global _site, _siteData informationFilePath = os.path.join(os.path.dirname(os.path.dirname(os.path.normpath(__file__))), "Site.json") # type: str try: with open(os.path.join(informationFilePath)) as informationFile: _site = Site(decoder.JSONDecoder().decode(informationFile.read())) except Exception as e: raise Exception("Failed to read site information for '" + informationFilePath + "'. \n") from e _siteData = { "Namespace": GetCurrentSite().Namespace, "Domain": GetCurrentSite().Domain, "GithubName": GetCurrentSite().GithubName, "BuildPath": Paths.BuildPath, "HostingPath": Paths.BuildPath }
def scrape_info(self, url): """ Scrapes post information for the given url Input: url : url of relevant post Output: df: dictionary containing relevant post information """ response = requests.get(url) soup = BeautifulSoup(response.text, "html.parser") decode = decoder.JSONDecoder() for i in soup.find_all("script", {"type":"text/javascript"}): if (re.findall('sharedData', str(i))): raw = re.sub('<.+?>', '', str(i)) without_header =raw[21:-1] content = decode.decode(without_header) post_content = content['entry_data']['PostPage'][0]['graphql']['shortcode_media'] taken_at = post_content['taken_at_timestamp'] number_of_likes = post_content['edge_media_preview_like']['count'] caption = post_content['edge_media_to_caption']['edges'][0]['node']['text'] number_of_comments = len(post_content['edge_media_to_comment']['edges']) people_tagged = [tagged['node']['user']['username'] for tagged in post_content['edge_media_to_tagged_user']['edges']] hashtags =[] comments = [] commenters=[] if post_content['edge_media_to_comment']['edges']: if post_content['edge_media_to_comment']['edges'][0]['node']['owner']['username'] == 'thedriftcollective': caption += (' ' + post_content['edge_media_to_comment']['edges'][0]['node']['text']) number_of_comments -= 1 for i in range(1, number_of_comments + 1): comments.append(post_content['edge_media_to_comment']['edges'][i]['node']['text']) commenters.append(post_content['edge_media_to_comment']['edges'][i]['node']['owner']['username']) else: comments = [item['node']['text'] for item in post_content['edge_media_to_comment']['edges']] commenters = [item['node']['owner']['username'] for item in post_content['edge_media_to_comment']['edges']] hashtags += [tag.strip() for tag in re.findall('\#.+?\s', post_content['edge_media_to_comment']['edges'][0]['node']['text'])] caption_hashtags = [tag.strip() for tag in re.findall('\#.+?\s', caption)] hashtags += caption_hashtags people_tagged += [re.sub('\@', '', tag) for tag in re.findall('\@.\w+', caption)] return {'taken_at':taken_at, 'url':url, 'number_of_likes' : number_of_likes, 'caption':caption, 'hashtags':hashtags, 'comments': comments, 'commenters' :commenters, 'people_tagged ':people_tagged , 'number_of_comments':number_of_comments }
def __call__(self, *args, **kwargs): # Caller can pass in a minimum value of timeout to be used for urlopen # call. Otherwise, the default socket timeout will be used. min_rpc_timeout = kwargs.pop('min_rpc_timeout', None) postdata = json_encoder_class().encode({'method': self.__serviceName, 'params': args + (kwargs,), 'id': 'jsonrpc'}) request = urllib2.Request(self.__serviceURL, data=postdata, headers=self.__headers) default_timeout = socket.getdefaulttimeout() if not default_timeout: # If default timeout is None, socket will never time out. respdata = urllib2.urlopen(request).read() else: timeout = max(min_rpc_timeout, default_timeout) respdata = urllib2.urlopen(request, timeout=timeout).read() try: resp = decoder.JSONDecoder().decode(respdata) except ValueError: raise JSONRPCException('Error decoding JSON reponse:\n' + respdata) if resp['error'] is not None: raise BuildException(resp['error']) else: return resp['result']
def __poll2_(self): """ 不知道干嘛的,一分钟连接一次,属于长连接,接收消息 @url:http://d.web2.qq.com/channel/poll2 r:{"clientid":"9467930","psessionid":"8368046764001e636f6e6e7365727665725f77656271714031302e3132382e36362e31313500003058000000c0026e040009456f266d0000000a407169446b464737436b6d00000028f8d256743e5c191cb40a2217845fab12fda62acd2e6145ae196976d7a8b3bb11a64d3c9565868322","key":0,"ids":[]} clientid:9467930 psessionid:8368046764001e636f6e6e7365727665725f77656271714031302e3132382e36362e31313500003058000000c0026e040009456f266d0000000a407169446b464737436b6d00000028f8d256743e5c191cb40a2217845fab12fda62acd2e6145ae196976d7a8b3bb11a64d3c9565868322 """ self.__headers.update({ 'Referer': 'http://d.web2.qq.com/proxy.html?v=20110331002&callback=2' }) urlv = 'http://d.web2.qq.com/channel/poll2' a = { 'clientid': self.__clientid, 'psessionid': self.__psessionid, 'key': 0, 'ids': [] } array = { 'r': json_encode.JSONEncoder().encode(a), 'clientid': self.__clientid, 'psessionid': self.__psessionid } self.__poll2 = self.__request(url=urlv, method='POST', data=array) str = json_decode.JSONDecoder().decode(self.__poll2) print(str) if str['retcode'] == 0: if str['result'][0]['poll_type'] == 'message': self.__message(str['result'][0]['value']['from_uin']) elif str['result'][0]['poll_type'] == 'group_message': self.__group_message(str['result'][0]['value']['from_uin']) pass t1 = th.Timer(1, self.__poll2_) t1.start() pass
def __call__(self, *args, **kwargs): # Caller can pass in a minimum value of timeout to be used for urlopen # call. Otherwise, the default socket timeout will be used. min_rpc_timeout = kwargs.pop('min_rpc_timeout', None) postdata = json_encoder_class().encode({'method': self.__serviceName, 'params': args + (kwargs,), 'id': 'jsonrpc'}) url_with_args = self.__serviceURL + '?' + urllib.urlencode({ 'method': self.__serviceName}) if self.__use_sso_client: respdata = _sso_request(url_with_args, self.__headers, postdata, min_rpc_timeout) else: respdata = _raw_http_request(url_with_args, self.__headers, postdata, min_rpc_timeout) try: resp = decoder.JSONDecoder().decode(respdata) except ValueError: raise JSONRPCException('Error decoding JSON reponse:\n' + respdata) if resp['error'] is not None: raise BuildException(resp['error']) else: return resp['result']
def GetIncludedPaths(strippingFilePath: str, targetDirectoryPath: str) -> typing.List[str]: includedPaths = list() # type: typing.List[str] with open(strippingFilePath) as strippingFile: strippingInformation = decoder.JSONDecoder().decode( strippingFile.read()) strippingPatterns = strippingInformation["Patterns"] # type: list if strippingInformation["Exclude Account Name"]: strippingPatterns.append(".*/" + getpass.getuser()) for strippingPatternIndex in range(0, len(strippingPatterns)): # type: int strippingPatterns[strippingPatternIndex] = re.compile( strippingPatterns[strippingPatternIndex]) for directoryRoot, directoryNames, fileNames in os.walk( targetDirectoryPath ): # type: str, typing.List[str], typing.List[str] for directoryName in directoryNames: if directoryRoot != targetDirectoryPath: directoryPath = os.path.join( directoryRoot.replace(targetDirectoryPath + os.path.sep, ""), directoryName).replace("\\", "/") # type: str else: directoryPath = directoryName # type: str pathExcluded = False # type: bool for strippingPattern in strippingPatterns: match = re.match(strippingPattern, directoryPath) if match is not None: pathExcluded = True break if not pathExcluded: includedPaths.append(directoryPath) for fileName in fileNames: if directoryRoot != targetDirectoryPath: filePath = os.path.join( directoryRoot.replace(targetDirectoryPath + os.path.sep, ""), fileName).replace("\\", "/") # type: str else: filePath = fileName # type: str pathExcluded = False # type: bool for strippingPattern in strippingPatterns: match = re.match(strippingPattern, filePath) if match is not None: pathExcluded = True break if not pathExcluded: includedPaths.append(filePath) return includedPaths
def _BuildModDownloads () -> None: from Automation import Distribution, Mods with open(os.path.join(Paths.TemplatesPath, "DownloadInstaller.html")) as downloadTemplateFile: downloadInstallerTemplate = downloadTemplateFile.read() with open(os.path.join(Paths.TemplatesPath, "DownloadInstallerAge.html")) as downloadTemplateFile: downloadInstallerAgeTemplate = downloadTemplateFile.read() with open(os.path.join(Paths.TemplatesPath, "DownloadFiles.html")) as downloadTemplateFile: downloadFilesTemplate = downloadTemplateFile.read() with open(os.path.join(Paths.TemplatesPath, "DownloadFilesAge.html")) as downloadTemplateFile: downloadFilesAgeTemplate = downloadTemplateFile.read() with open(os.path.join(Paths.TemplatesPath, "DownloadSources.html")) as downloadTemplateFile: downloadSourcesTemplate = downloadTemplateFile.read() with open(os.path.join(Paths.TemplatesPath, "DownloadSourcesAge.html")) as downloadTemplateFile: downloadSourcesAgeTemplate = downloadTemplateFile.read() for modNamespace, modVersions in Distribution.Releases.items(): # type: str, typing.List[Distribution.ModVersion] modBuildPath = os.path.join(Paths.DownloadsBuildPath, "mods", modNamespace.lower()) # type: str licensePath = os.path.join(modBuildPath.replace(Paths.DownloadsBuildPath + os.path.sep, ""), _licenseName) # type: str licensePath = licensePath.replace("\\", "/") with open(os.path.join(Paths.DownloadsSourcesPath, "mods", modNamespace + ".json")) as modInformationFile: modInformation = decoder.JSONDecoder().decode(modInformationFile.read()) # type: dict if not modInformation["Mod"]["Age"]: modInstallerTemplate = downloadInstallerTemplate # type: str modFilesTemplate = downloadFilesTemplate # type: str else: modInstallerTemplate = downloadInstallerAgeTemplate # type: str modFilesTemplate = downloadFilesAgeTemplate # type: str if not modInformation["Sources"]["Age"]: sourcesTemplate = downloadSourcesTemplate # type: str else: sourcesTemplate = downloadSourcesAgeTemplate # type: str mod = Mods.GetMod(modNamespace) # type: Mods.Mod modLatestVersion = mod.ReleaseLatest # type: Distribution.ModVersion modName = mod.GetName() # type: typing.Optional[str] if modName is None: modName = modNamespace latestBasePath = os.path.relpath(Paths.DownloadsBuildPath, modBuildPath).replace("\\", "/") + "/.." # type: str latestInstallerPath = os.path.join(modBuildPath, "installer/index.html") # type: str latestInstallerURL = modLatestVersion.InstallerURL # type: str latestFilesPath = os.path.join(modBuildPath, "files/index.html") # type: str latestFilesRelativePath = os.path.relpath(latestFilesPath, Paths.DownloadsBuildPath) # type: str latestFilesURL = modLatestVersion.FilesURL # type: str latestSourcesPath = os.path.join(modBuildPath, "sources/index.html") # type: str latestSourcesURL = modLatestVersion.SourcesURL # type: str _WriteDownload(latestInstallerPath, modInstallerTemplate, latestBasePath, modName, str(modLatestVersion.Version), _typeInstaller, str(modLatestVersion.GameVersion), modLatestVersion.ReleaseDate, latestFilesRelativePath, licensePath, latestInstallerURL) _WriteDownload(latestFilesPath, modFilesTemplate, latestBasePath, modName, str(modLatestVersion.Version), _typeFiles, str(modLatestVersion.GameVersion), modLatestVersion.ReleaseDate, latestFilesRelativePath, licensePath, latestFilesURL) _WriteDownload(latestSourcesPath, sourcesTemplate, latestBasePath, modName, str(modLatestVersion.Version), _typeSources, str(modLatestVersion.GameVersion), modLatestVersion.ReleaseDate, latestFilesRelativePath, licensePath, latestSourcesURL) for modVersion in modVersions: # type: Distribution.ModVersion versionBuildPath = os.path.join(modBuildPath, str(modVersion.Version)) basePath = os.path.relpath(Paths.DownloadsBuildPath, versionBuildPath).replace("\\", "/") + "/.." # type: str installerPath = os.path.join(versionBuildPath, "installer/index.html") # type: str installerURL = modVersion.InstallerURL # type: str filesPath = os.path.join(versionBuildPath, "files/index.html") # type: str filesRelativePath = os.path.relpath(filesPath, Paths.DownloadsBuildPath) # type: str filesURL = modVersion.FilesURL # type: str sourcesPath = os.path.join(versionBuildPath, "sources/index.html") # type: str sourcesURL = modVersion.SourcesURL # type: str _WriteDownload(installerPath, modInstallerTemplate, basePath, modName, str(modVersion.Version), _typeInstaller, str(modLatestVersion.GameVersion), modLatestVersion.ReleaseDate, filesRelativePath, licensePath, installerURL) _WriteDownload(filesPath, modFilesTemplate, basePath, modName, str(modVersion.Version), _typeFiles, str(modVersion.GameVersion), modVersion.ReleaseDate, filesRelativePath, licensePath, filesURL) _WriteDownload(sourcesPath, sourcesTemplate, basePath, modName, str(modVersion.Version), _typeSources, str(modVersion.GameVersion), modVersion.ReleaseDate, filesRelativePath, licensePath, sourcesURL) for modNamespace, modVersions in Distribution.Previews.items(): # type: str, typing.List[Distribution.ModVersion] modBuildPath = os.path.join(Paths.DownloadsBuildPath, "Mods", modNamespace) # type: str licensePath = os.path.join(modBuildPath.replace(Paths.DownloadsBuildPath + os.path.sep, ""), _licenseName) # type: str licensePath = licensePath.replace("\\", "/") with open(os.path.join(Paths.DownloadsSourcesPath, "mods", modNamespace + ".json")) as modInformationFile: modInformation = decoder.JSONDecoder().decode(modInformationFile.read()) # type: dict if not modInformation["Mod"]["Age"]: modInstallerTemplate = downloadInstallerTemplate # type: str modFilesTemplate = downloadFilesTemplate # type: str else: modInstallerTemplate = downloadInstallerAgeTemplate # type: str modFilesTemplate = downloadFilesAgeTemplate # type: str if not modInformation["Sources"]["Age"]: sourcesTemplate = downloadSourcesTemplate # type: str else: sourcesTemplate = downloadSourcesAgeTemplate # type: str mod = Mods.GetMod(modNamespace) # type: Mods.Mod modName = mod.GetName() # type: typing.Optional[str] for modVersion in modVersions: # type: Distribution.ModVersion versionBuildPath = os.path.join(modBuildPath, str(modVersion.Version), modVersion.ConcealerFolderName) basePath = os.path.relpath(Paths.DownloadsBuildPath, versionBuildPath).replace("\\", "/") + "/.." # type: str installerPath = os.path.join(versionBuildPath, "installer/index.html") # type: str installerURL = modVersion.InstallerURL # type: str filesPath = os.path.join(versionBuildPath, "files/index.html") # type: str filesRelativePath = os.path.relpath(filesPath, Paths.DownloadsBuildPath) # type: str filesURL = modVersion.FilesURL # type: str sourcesPath = os.path.join(versionBuildPath, "sources/index.html") # type: str sourcesURL = modVersion.SourcesURL # type: str _WriteDownload(installerPath, modInstallerTemplate, basePath, modName, str(modVersion.Version), _typeFiles, str(modVersion.GameVersion), modVersion.ReleaseDate, filesRelativePath, licensePath, installerURL) _WriteDownload(filesPath, modFilesTemplate, basePath, modName, str(modVersion.Version), _typeFiles, str(modVersion.GameVersion), modVersion.ReleaseDate, filesRelativePath, licensePath, filesURL) _WriteDownload(sourcesPath, sourcesTemplate, basePath, modName, str(modVersion.Version), _typeSources, str(modVersion.GameVersion), modVersion.ReleaseDate, filesRelativePath, licensePath, sourcesURL)
def __init__(self, applicationPath: str): self.Name = os.path.split(applicationPath)[1] # type: str self.PointerDirectoryPath = applicationPath # type: str self.Special = "" # type: str with open(os.path.join(self.PointerDirectoryPath, "Application.json")) as applicationPointerFile: applicationPointerDictionary = decoder.JSONDecoder().decode( applicationPointerFile.read() ) # type: typing.Dict[str, typing.Any] special = applicationPointerDictionary.get("Special") # type: str if isinstance(special, str): self.Special = special self.ExecutablePath = None # type: typing.Union[str, None] self.ExecutableSpecial = None # type: typing.Union[str, None] if "Paths" in applicationPointerDictionary: for pathDictionary in applicationPointerDictionary[ "Paths"]: # type: typing.Dict[str, str] executableFullFilePath = None # type: typing.Optional[str] executableRegistryKey = pathDictionary.get( "Registry Key") # type: str executableFilePath = pathDictionary.get( "File Path") # type: str executableSpecial = pathDictionary.get("Special") # type: str registryKeyPath = None # type: typing.Optional[str] if isinstance(executableRegistryKey, str): executableRegistryKey = executableRegistryKey.replace( "/", "\\") if executableRegistryKey.count("") < 1: continue if platform.system() == "Windows": try: registryKeyPath = Registry.ReadRegistryFullKey( executableRegistryKey) except: pass if not isinstance(registryKeyPath, str) and registryKeyPath is not None: raise Exception( "Application registry key value is not a string" ) if registryKeyPath is not None: if isinstance(executableFilePath, str): executableFullFilePath = os.path.normpath( os.path.join(registryKeyPath, executableFilePath)) else: executableFullFilePath = os.path.normpath( registryKeyPath) pass else: if isinstance(executableFilePath, str): executableFullFilePath = os.path.normpath( os.path.join(self.PointerDirectoryPath, os.path.normpath(executableFilePath))) if executableFullFilePath is not None: if os.path.exists(executableFullFilePath): self.ExecutablePath = executableFullFilePath self.ExecutableSpecial = executableSpecial if self.ExecutablePath is None: print("Cannot find valid path to application '" + self.Name + "'.", file=sys.stderr)
def BuildPackageChanges() -> bool: if not Package.CanBuildPackage(): return False for package in Mod.GetCurrentMod().Packages: # type: Mod.Package baseFileExists = os.path.exists( package.SourceBaseFilePath) # type: bool loosePathExists = os.path.exists(package.SourceLoosePath) # type: bool packageManifest = None # type: typing.Optional[typing.Dict[str, typing.Union[float, typing.Dict[str, float]]]] if not os.path.exists(package.BuildFilePath): _BuildPackageEverythingInternal(package) return True if os.path.exists(package.BuildManifestFilePath): with open(package.BuildManifestFilePath) as packageManifestFile: packageManifest = decoder.JSONDecoder().decode( packageManifestFile.read()) if packageManifest is not None and not isinstance( packageManifest, dict): packageManifest = None if packageManifest is not None and (not "Loose" in packageManifest or not "Base" in packageManifest): packageManifest = None if packageManifest is None: _BuildPackageEverythingInternal(package) return True else: filesChanged = False # type: bool if baseFileExists: baseCurrentChangeTime = os.path.getmtime( package.SourceBaseFilePath) # type: float if packageManifest["Base"] != os.path.getmtime( package.SourceBaseFilePath): packageManifest["Base"] = baseCurrentChangeTime filesChanged = True if loosePathExists: packageManifestLooseDictionary = packageManifest[ "Loose"] # type: dict for entryFileName in list( packageManifestLooseDictionary.keys()): # type: str entryChangeTime = packageManifestLooseDictionary[ entryFileName] # type: float entryFilePath = os.path.join(package.SourceLoosePath, entryFileName) # type: str if not os.path.exists(entryFilePath): packageManifestLooseDictionary.pop(entryFileName) filesChanged = True continue entryCurrentChangeTime = os.path.getmtime( entryFilePath) # type: float if entryCurrentChangeTime != entryChangeTime: packageManifest["Loose"][ entryFileName] = entryCurrentChangeTime filesChanged = True for sourceDirectoryRoot, sourceDirectoryNames, sourceFileNames in os.walk( package.SourceLoosePath ): # type: str, typing.List[str], typing.List[str] for sourceFileName in sourceFileNames: # type: str sourceFilePath = os.path.join( sourceDirectoryRoot, sourceFileName) # type: str relativeSourceFilePath = os.path.relpath( sourceFilePath, package.SourceLoosePath) # type: str sourceFileDuplicate = False # type: bool for entryFileName in packageManifest["Loose"].keys( ): # type: str if entryFileName.lower( ) == relativeSourceFilePath.lower(): sourceFileDuplicate = True break if not sourceFileDuplicate: packageManifest["Loose"][ relativeSourceFilePath] = os.path.getmtime( sourceFilePath) filesChanged = True if filesChanged: addingFilePaths = list() # type: typing.List[str] if loosePathExists: for sourceDirectoryRoot, sourceDirectoryNames, sourceFileNames in os.walk( package.SourceLoosePath ): # type: str, typing.List[str], typing.List[str] for sourceFileName in sourceFileNames: # type: str # noinspection SpellCheckingInspection if os.path.splitext(sourceFileName)[1].lower( ) == ".sourceinfo": continue sourceFilePath = os.path.join( sourceDirectoryRoot, sourceFileName) # type: str if os.path.isfile(sourceFilePath): addingFilePaths.append(sourceFilePath) if baseFileExists: Package.BuildPackage( package.BuildFilePath, baseFilePath=package.SourceBaseFilePath, addingFilePaths=addingFilePaths) else: Package.BuildPackage(package.BuildFilePath, addingFilePaths=addingFilePaths) with open(package.BuildManifestFilePath, "w+") as packageManifestFile: packageManifestFile.write( encoder.JSONEncoder( indent="\t").encode(packageManifest)) return True
import socket import traceback from json import decoder try: from django.core import exceptions as django_exceptions # Django JSON encoder uses the standard json encoder but can handle DateTime from django.core.serializers import json as django_encoder json_encoder = django_encoder.DjangoJSONEncoder() except django_exceptions.ImproperlyConfigured: from json import encoder json_encoder = encoder.JSONEncoder() json_decoder = decoder.JSONDecoder() def customConvertJson(value): """\ Recursively process JSON values and do type conversions. -change floats to ints -change unicodes to strs """ if isinstance(value, float): return int(value) elif isinstance(value, unicode): return str(value) elif isinstance(value, list): return [customConvertJson(item) for item in value] elif isinstance(value, dict):