def _ReadDocumentConfig(documentConfigFilePath: str) -> typing.Tuple[str, str]:
"""
:return:
Document build path,
Document text,
Indexing information
"""
with open(documentConfigFilePath) as documentConfigFile:
documentConfig = decoder.JSONDecoder().decode(
documentConfigFile.read()) # type: dict
documentTemplateConfigFilePath = os.path.join(
Paths.DocumentsConfigTemplatesPath,
documentConfig["TemplateConfig"]) # type: str
with open(documentTemplateConfigFilePath) as documentTemplateConfigFile:
documentTemplateConfig = decoder.JSONDecoder().decode(
documentTemplateConfigFile.read()) # type: dict
documentFileExtension = documentConfig["FileExtension"] # type: str
documentRelativeFilePath = os.path.splitext(
documentConfigFilePath.replace(
Paths.DocumentsConfigDocumentsPath + os.path.sep,
""))[0] + "." + documentFileExtension # type: str
documentRelativeFilePath.replace(os.path.altsep, os.path.sep)
documentFilePath = os.path.join(Paths.DocumentsBuildPath,
documentRelativeFilePath)
documentAdditionalValues = {"DocumentPath": documentRelativeFilePath}
documentValues = _ReadValues(documentConfig,
documentAdditionalValues) # type: dict
documentValues["DocumentPath"] = documentRelativeFilePath
if documentValues.get("BasePath") is None:
documentRelativeFilePathParts = documentRelativeFilePath.split(
os.path.sep) # type: typing.List[str]
documentBasePath = "../" * (len(documentRelativeFilePathParts) - 1)
documentValues["BasePath"] = documentBasePath
documentText = _ReadDocument(
documentValues, documentTemplateConfig["Document"]) # type: str
return documentFilePath, documentText
def __request_post(self):
'''
http://d.web2.qq.com/channel/login2
r:{"status":"online","ptwebqq":"95b148b95af9be7677757b3a629e3904f52f153d0b714c527f81f8d9e385867a","passwd_sig":"",
"clientid":"21628014","psessionid":null}
clientid:21628014
psessionid:null
'''
self.__headers.update({
'Referer':
'http://d.web2.qq.com/proxy.html?v=20110331002&callback=2'
})
a = {
'status': 'online',
'ptwebqq': self.__getcookies('ptwebqq'),
'passwd_sig': '',
'clientid': self.__clientid,
'psessionid': 'null'
}
array = {
'r': json_encode.JSONEncoder().encode(a),
'clientid': self.__clientid,
'psessionid': 'null'
}
url = 'http://d.web2.qq.com/channel/login2'
str = self.__request(url, 'POST', array)
str = json_decode.JSONDecoder().decode(str)
self.__psessionid = str['result']['psessionid']
self.__vfwebqq = str['result']['vfwebqq']
self.__get_friend_info2()
self.__get_user_friends2()
self.__get_group_name_list_mask2()
self.__poll2_()
self.__get_msg_tip_()
pass
def _Setup() -> None:
global _mod, _modData
informationFilePath = os.path.join(
os.path.dirname(os.path.dirname(os.path.normpath(__file__))),
"Mod.json") # type: str
try:
with open(os.path.join(informationFilePath)) as informationFile:
_mod = Mod(decoder.JSONDecoder().decode(informationFile.read()))
except Exception as e:
raise Exception("Failed to read mod information for '" +
informationFilePath + "'. \n") from e
_modData = {
"Namespace": GetCurrentMod().Namespace,
"Name": GetCurrentMod().Name,
"Version": GetCurrentMod().Version,
"ChangesFilePath": GetCurrentMod().ChangesFilePath,
"PlansFilePath": GetCurrentMod().PlansFilePath,
"InstallerFilePath": GetCurrentMod().DistributionInstallerFilePath,
"FilesFilePath": GetCurrentMod().DistributionFilesFilePath,
"SourcesFileName": GetCurrentMod().SourcesFileName,
"GithubName": GetCurrentMod().GithubName
} # type: typing.Dict[str, typing.Any]
def scrape_shortcodes(self, initial_url, ):
""" Scrapes shortcodes contained in initial_url (ie. the redirect information for urls of the page's posts)
Input:
initial_url : url to initial page
Output:
df: pandas DataFrame with relevant information from scraped posts
"""
is_more = True
shortcodes = []
next_url = initial_url
while is_more:
response = requests.get(next_url)
print(response.status_code)
soup = BeautifulSoup(response.text, "html.parser")
decode = decoder.JSONDecoder()
home_page = decode.decode(str(soup))
for i in range(len(home_page['data']['user']['edge_owner_to_timeline_media']['edges'])):
sc = home_page['data']['user']['edge_owner_to_timeline_media']['edges'][i]['node']['shortcode']
shortcodes.append(sc)
time.sleep(self.sleep)
if home_page['data']['user']['edge_owner_to_timeline_media']['page_info']['has_next_page'] == True:
nextpage = home_page['data']['user']['edge_owner_to_timeline_media']['page_info']['end_cursor']
next_url = 'https://www.instagram.com/graphql/query/?query_id=17888483320059182&variables=%7B%22id%22%3A%222183125078%22%2C%22first%22%3A12%2C%22after%22%3A%22{}%22%7D'.format(nextpage)
else:
is_more = False
self.shortcodes = shortcodes
def extended_json_decoder():
'''Return a decoder for the superset of JSON understood by the
upatch function.
The exact nature of this superset is documented in the manpage for
json_patch(1). Briefly, the string ``...`` is accepted where
standard JSON expects a ``{<property name>}: {<object>}``
construction, and the string ``...``, optionally followed by a
number in parentheses, is accepted where standard JSON expects an
array element.
The superset is implemented by parsing ``...`` in JSON objects as
a key/value pair ``Ellipsis: True`` in the resulting dictionary,
and ``...({num}){?}`` as a subsequence of ``{num}`` ``Ellipsis``
objects, or one ellipsis object if ``({num})`` is not present.
Examples:
>>> dec = extended_json_decoder()
>>> (dec.decode('{"foo": "bar",...,"baz": "quux"}') ==
... {"foo": "bar", "baz": "quux", Ellipsis: True})
True
>>> dec.decode('[...]')
[Ellipsis]
>>> dec.decode('["foo",...(3),"bar"]')
[u'foo', Ellipsis, Ellipsis, Ellipsis, u'bar']
'''
dec = decoder.JSONDecoder()
dec.parse_object = _JSONObject
dec.parse_array = _JSONArray
dec.scan_once = scanner.py_make_scanner(dec)
return dec
def _SetupExternalPaths() -> None:
global DistributionReleasesPath, DistributionPreviewsPath, WebsitesDistributionPath
with open(os.path.join(SettingsPath,
"ExternalPaths.json")) as externalPathsFile:
externalPathsInformation = decoder.JSONDecoder().decode(
externalPathsFile.read())
distributionReleasesRelativePath = externalPathsInformation.get(
"DistributionReleasesPath", None) # type: str
if distributionReleasesRelativePath is not None:
DistributionReleasesPath = os.path.normpath(
os.path.abspath(
os.path.join(AutomationPath,
distributionReleasesRelativePath)))
distributionPreviewsRelativePath = externalPathsInformation.get(
"DistributionPreviewsPath", None) # type: str
if distributionPreviewsRelativePath is not None:
DistributionPreviewsPath = os.path.normpath(
os.path.abspath(
os.path.join(AutomationPath,
distributionPreviewsRelativePath)))
websitesDistributionRelativePath = externalPathsInformation.get(
"WebsitesDistributionPath", None) # type: str
if websitesDistributionRelativePath is not None:
WebsitesDistributionPath = os.path.normpath(
os.path.abspath(
os.path.join(AutomationPath,
websitesDistributionRelativePath)))
def test_json_object():
pairs, end = parse_json_object(('"abc":213}', 0), 'utf-8')
print pairs, type(pairs)
jd = decoder.JSONDecoder()
pairs, end = decoder.JSONObject(('"abc":213}', 0), 'utf-8', scan_once = jd.scan_once, strict=False, object_hook=None, object_pairs_hook=None)
print pairs, type(pairs)
p = []
print type(p)
p.append(('abc', 213))
print type(p)
def __init__(self):
self.__socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
with open("config.conf", 'r') as content_file:
decoder = decoder_json.JSONDecoder()
decoded = decoder.decode(content_file.read())
host = decoded['host']
port = int(decoded['port'])
server_address = (host, port)
self.__socket.bind(server_address)
# listen() puts the socket into server mode
self.__socket.listen(1000)
from Server.controllers.RoomsController import RoomsController
self.__room_controller_thread = RoomsController()
def read_config() -> dict:
if os.path.exists("config.json"):
cfg_decoder = decoder.JSONDecoder()
with open("config.json", 'rt') as f:
return cfg_decoder.decode('\n'.join(f.readlines()))
else:
print('Config file not found. Creating new one...')
cfg = create_default_config()
cfg_encoder = encoder.JSONEncoder()
json = cfg_encoder.encode(cfg)
with open("config.json", 'wt') as f:
f.write(json)
f.flush()
return cfg
def GetModVersion(self) -> str:
with open(self.InformationSourceFilePath) as informationFile:
informationDictionary = decoder.JSONDecoder().decode(
informationFile.read()) # type: typing.Dict[str, typing.Any]
if not "Version" in informationDictionary:
raise ValueError("Entry 'Version' does not exist.")
version = informationDictionary["Version"]
if not isinstance(version, str):
raise TypeError("Entry 'Version' is not a string.")
return version
def parse_rate(self, response):
item = SupItem()
item['website'] = response.meta['link']
item['word'] = response.meta['word']
json = response.body[response.body.find('{'):-2]
print 'json->>>>>>>>>>>>>>>>>>>>>>>', json
jsonD = decoder.JSONDecoder()
djson = jsonD.decode(json)
item['lastOneWeek'] = djson['data']['starStatisticsList'][5]['weekly']
item['lastOneMonth'] = djson['data']['starStatisticsList'][5][
'monthly']
item['lastSixMonth'] = djson['data']['starStatisticsList'][5][
'halfYearly']
item['beforeHalfYear'] = djson['data']['starStatisticsList'][5][
'halfYearBefore']
item['total'] = djson['data']['starStatisticsList'][5]['totals']
return item
def __call__(self, *args, **kwargs):
postdata = json_encoder_class().encode({
'method': self.__serviceName,
'params': args + (kwargs, ),
'id': 'jsonrpc'
})
request = urllib2.Request(self.__serviceURL,
data=postdata,
headers=self.__headers)
respdata = urllib2.urlopen(request).read()
try:
resp = decoder.JSONDecoder().decode(respdata)
except ValueError:
raise JSONRPCException('Error decoding JSON reponse:\n' + respdata)
if resp['error'] is not None:
raise BuildException(resp['error'])
else:
return resp['result']
def iter_response_objects(response):
prev = ''
decoder = json_decoder.JSONDecoder()
for chunk in response.read_chunked(decode_content=False):
if isinstance(chunk, bytes):
chunk = chunk.decode('utf-8')
chunk = prev + chunk
length = len(chunk)
idx = json_decoder.WHITESPACE.match(chunk, 0).end()
while idx < length:
try:
message, idx = decoder.raw_decode(chunk, idx)
except ValueError: # malformed or incomplete JSON, unlikely to happen
break
else:
yield message
idx = json_decoder.WHITESPACE.match(chunk, idx).end()
prev = chunk[idx:]
def _Setup () -> None:
global _site, _siteData
informationFilePath = os.path.join(os.path.dirname(os.path.dirname(os.path.normpath(__file__))), "Site.json") # type: str
try:
with open(os.path.join(informationFilePath)) as informationFile:
_site = Site(decoder.JSONDecoder().decode(informationFile.read()))
except Exception as e:
raise Exception("Failed to read site information for '" + informationFilePath + "'. \n") from e
_siteData = {
"Namespace": GetCurrentSite().Namespace,
"Domain": GetCurrentSite().Domain,
"GithubName": GetCurrentSite().GithubName,
"BuildPath": Paths.BuildPath,
"HostingPath": Paths.BuildPath
}
def scrape_info(self, url):
""" Scrapes post information for the given url
Input:
url : url of relevant post
Output:
df: dictionary containing relevant post information
"""
response = requests.get(url)
soup = BeautifulSoup(response.text, "html.parser")
decode = decoder.JSONDecoder()
for i in soup.find_all("script", {"type":"text/javascript"}):
if (re.findall('sharedData', str(i))):
raw = re.sub('<.+?>', '', str(i))
without_header =raw[21:-1]
content = decode.decode(without_header)
post_content = content['entry_data']['PostPage'][0]['graphql']['shortcode_media']
taken_at = post_content['taken_at_timestamp']
number_of_likes = post_content['edge_media_preview_like']['count']
caption = post_content['edge_media_to_caption']['edges'][0]['node']['text']
number_of_comments = len(post_content['edge_media_to_comment']['edges'])
people_tagged = [tagged['node']['user']['username'] for tagged in post_content['edge_media_to_tagged_user']['edges']]
hashtags =[]
comments = []
commenters=[]
if post_content['edge_media_to_comment']['edges']:
if post_content['edge_media_to_comment']['edges'][0]['node']['owner']['username'] == 'thedriftcollective':
caption += (' ' + post_content['edge_media_to_comment']['edges'][0]['node']['text'])
number_of_comments -= 1
for i in range(1, number_of_comments + 1):
comments.append(post_content['edge_media_to_comment']['edges'][i]['node']['text'])
commenters.append(post_content['edge_media_to_comment']['edges'][i]['node']['owner']['username'])
else:
comments = [item['node']['text'] for item in post_content['edge_media_to_comment']['edges']]
commenters = [item['node']['owner']['username'] for item in post_content['edge_media_to_comment']['edges']]
hashtags += [tag.strip() for tag in re.findall('\#.+?\s', post_content['edge_media_to_comment']['edges'][0]['node']['text'])]
caption_hashtags = [tag.strip() for tag in re.findall('\#.+?\s', caption)]
hashtags += caption_hashtags
people_tagged += [re.sub('\@', '', tag) for tag in re.findall('\@.\w+', caption)]
return {'taken_at':taken_at, 'url':url, 'number_of_likes' : number_of_likes, 'caption':caption, 'hashtags':hashtags, 'comments': comments, 'commenters' :commenters, 'people_tagged ':people_tagged , 'number_of_comments':number_of_comments }
def __call__(self, *args, **kwargs):
# Caller can pass in a minimum value of timeout to be used for urlopen
# call. Otherwise, the default socket timeout will be used.
min_rpc_timeout = kwargs.pop('min_rpc_timeout', None)
postdata = json_encoder_class().encode({'method': self.__serviceName,
'params': args + (kwargs,),
'id': 'jsonrpc'})
request = urllib2.Request(self.__serviceURL, data=postdata,
headers=self.__headers)
default_timeout = socket.getdefaulttimeout()
if not default_timeout:
# If default timeout is None, socket will never time out.
respdata = urllib2.urlopen(request).read()
else:
timeout = max(min_rpc_timeout, default_timeout)
respdata = urllib2.urlopen(request, timeout=timeout).read()
try:
resp = decoder.JSONDecoder().decode(respdata)
except ValueError:
raise JSONRPCException('Error decoding JSON reponse:\n' + respdata)
if resp['error'] is not None:
raise BuildException(resp['error'])
else:
return resp['result']
def __poll2_(self):
"""
不知道干嘛的，一分钟连接一次，属于长连接，接收消息
@url:http://d.web2.qq.com/channel/poll2
r:{"clientid":"9467930","psessionid":"8368046764001e636f6e6e7365727665725f77656271714031302e3132382e36362e31313500003058000000c0026e040009456f266d0000000a407169446b464737436b6d00000028f8d256743e5c191cb40a2217845fab12fda62acd2e6145ae196976d7a8b3bb11a64d3c9565868322","key":0,"ids":[]}
clientid:9467930
psessionid:8368046764001e636f6e6e7365727665725f77656271714031302e3132382e36362e31313500003058000000c0026e040009456f266d0000000a407169446b464737436b6d00000028f8d256743e5c191cb40a2217845fab12fda62acd2e6145ae196976d7a8b3bb11a64d3c9565868322
"""
self.__headers.update({
'Referer':
'http://d.web2.qq.com/proxy.html?v=20110331002&callback=2'
})
urlv = 'http://d.web2.qq.com/channel/poll2'
a = {
'clientid': self.__clientid,
'psessionid': self.__psessionid,
'key': 0,
'ids': []
}
array = {
'r': json_encode.JSONEncoder().encode(a),
'clientid': self.__clientid,
'psessionid': self.__psessionid
}
self.__poll2 = self.__request(url=urlv, method='POST', data=array)
str = json_decode.JSONDecoder().decode(self.__poll2)
print(str)
if str['retcode'] == 0:
if str['result'][0]['poll_type'] == 'message':
self.__message(str['result'][0]['value']['from_uin'])
elif str['result'][0]['poll_type'] == 'group_message':
self.__group_message(str['result'][0]['value']['from_uin'])
pass
t1 = th.Timer(1, self.__poll2_)
t1.start()
pass
def __call__(self, *args, **kwargs):
# Caller can pass in a minimum value of timeout to be used for urlopen
# call. Otherwise, the default socket timeout will be used.
min_rpc_timeout = kwargs.pop('min_rpc_timeout', None)
postdata = json_encoder_class().encode({'method': self.__serviceName,
'params': args + (kwargs,),
'id': 'jsonrpc'})
url_with_args = self.__serviceURL + '?' + urllib.urlencode({
'method': self.__serviceName})
if self.__use_sso_client:
respdata = _sso_request(url_with_args, self.__headers, postdata,
min_rpc_timeout)
else:
respdata = _raw_http_request(url_with_args, self.__headers,
postdata, min_rpc_timeout)
try:
resp = decoder.JSONDecoder().decode(respdata)
except ValueError:
raise JSONRPCException('Error decoding JSON reponse:\n' + respdata)
if resp['error'] is not None:
raise BuildException(resp['error'])
else:
return resp['result']
def GetIncludedPaths(strippingFilePath: str,
targetDirectoryPath: str) -> typing.List[str]:
includedPaths = list() # type: typing.List[str]
with open(strippingFilePath) as strippingFile:
strippingInformation = decoder.JSONDecoder().decode(
strippingFile.read())
strippingPatterns = strippingInformation["Patterns"] # type: list
if strippingInformation["Exclude Account Name"]:
strippingPatterns.append(".*/" + getpass.getuser())
for strippingPatternIndex in range(0, len(strippingPatterns)): # type: int
strippingPatterns[strippingPatternIndex] = re.compile(
strippingPatterns[strippingPatternIndex])
for directoryRoot, directoryNames, fileNames in os.walk(
targetDirectoryPath
): # type: str, typing.List[str], typing.List[str]
for directoryName in directoryNames:
if directoryRoot != targetDirectoryPath:
directoryPath = os.path.join(
directoryRoot.replace(targetDirectoryPath + os.path.sep,
""),
directoryName).replace("\\", "/") # type: str
else:
directoryPath = directoryName # type: str
pathExcluded = False # type: bool
for strippingPattern in strippingPatterns:
match = re.match(strippingPattern, directoryPath)
if match is not None:
pathExcluded = True
break
if not pathExcluded:
includedPaths.append(directoryPath)
for fileName in fileNames:
if directoryRoot != targetDirectoryPath:
filePath = os.path.join(
directoryRoot.replace(targetDirectoryPath + os.path.sep,
""),
fileName).replace("\\", "/") # type: str
else:
filePath = fileName # type: str
pathExcluded = False # type: bool
for strippingPattern in strippingPatterns:
match = re.match(strippingPattern, filePath)
if match is not None:
pathExcluded = True
break
if not pathExcluded:
includedPaths.append(filePath)
return includedPaths
def _BuildModDownloads () -> None:
from Automation import Distribution, Mods
with open(os.path.join(Paths.TemplatesPath, "DownloadInstaller.html")) as downloadTemplateFile:
downloadInstallerTemplate = downloadTemplateFile.read()
with open(os.path.join(Paths.TemplatesPath, "DownloadInstallerAge.html")) as downloadTemplateFile:
downloadInstallerAgeTemplate = downloadTemplateFile.read()
with open(os.path.join(Paths.TemplatesPath, "DownloadFiles.html")) as downloadTemplateFile:
downloadFilesTemplate = downloadTemplateFile.read()
with open(os.path.join(Paths.TemplatesPath, "DownloadFilesAge.html")) as downloadTemplateFile:
downloadFilesAgeTemplate = downloadTemplateFile.read()
with open(os.path.join(Paths.TemplatesPath, "DownloadSources.html")) as downloadTemplateFile:
downloadSourcesTemplate = downloadTemplateFile.read()
with open(os.path.join(Paths.TemplatesPath, "DownloadSourcesAge.html")) as downloadTemplateFile:
downloadSourcesAgeTemplate = downloadTemplateFile.read()
for modNamespace, modVersions in Distribution.Releases.items(): # type: str, typing.List[Distribution.ModVersion]
modBuildPath = os.path.join(Paths.DownloadsBuildPath, "mods", modNamespace.lower()) # type: str
licensePath = os.path.join(modBuildPath.replace(Paths.DownloadsBuildPath + os.path.sep, ""), _licenseName) # type: str
licensePath = licensePath.replace("\\", "/")
with open(os.path.join(Paths.DownloadsSourcesPath, "mods", modNamespace + ".json")) as modInformationFile:
modInformation = decoder.JSONDecoder().decode(modInformationFile.read()) # type: dict
if not modInformation["Mod"]["Age"]:
modInstallerTemplate = downloadInstallerTemplate # type: str
modFilesTemplate = downloadFilesTemplate # type: str
else:
modInstallerTemplate = downloadInstallerAgeTemplate # type: str
modFilesTemplate = downloadFilesAgeTemplate # type: str
if not modInformation["Sources"]["Age"]:
sourcesTemplate = downloadSourcesTemplate # type: str
else:
sourcesTemplate = downloadSourcesAgeTemplate # type: str
mod = Mods.GetMod(modNamespace) # type: Mods.Mod
modLatestVersion = mod.ReleaseLatest # type: Distribution.ModVersion
modName = mod.GetName() # type: typing.Optional[str]
if modName is None:
modName = modNamespace
latestBasePath = os.path.relpath(Paths.DownloadsBuildPath, modBuildPath).replace("\\", "/") + "/.." # type: str
latestInstallerPath = os.path.join(modBuildPath, "installer/index.html") # type: str
latestInstallerURL = modLatestVersion.InstallerURL # type: str
latestFilesPath = os.path.join(modBuildPath, "files/index.html") # type: str
latestFilesRelativePath = os.path.relpath(latestFilesPath, Paths.DownloadsBuildPath) # type: str
latestFilesURL = modLatestVersion.FilesURL # type: str
latestSourcesPath = os.path.join(modBuildPath, "sources/index.html") # type: str
latestSourcesURL = modLatestVersion.SourcesURL # type: str
_WriteDownload(latestInstallerPath, modInstallerTemplate, latestBasePath,
modName, str(modLatestVersion.Version), _typeInstaller, str(modLatestVersion.GameVersion), modLatestVersion.ReleaseDate,
latestFilesRelativePath, licensePath, latestInstallerURL)
_WriteDownload(latestFilesPath, modFilesTemplate, latestBasePath,
modName, str(modLatestVersion.Version), _typeFiles, str(modLatestVersion.GameVersion), modLatestVersion.ReleaseDate,
latestFilesRelativePath, licensePath, latestFilesURL)
_WriteDownload(latestSourcesPath, sourcesTemplate, latestBasePath,
modName, str(modLatestVersion.Version), _typeSources, str(modLatestVersion.GameVersion), modLatestVersion.ReleaseDate,
latestFilesRelativePath, licensePath, latestSourcesURL)
for modVersion in modVersions: # type: Distribution.ModVersion
versionBuildPath = os.path.join(modBuildPath, str(modVersion.Version))
basePath = os.path.relpath(Paths.DownloadsBuildPath, versionBuildPath).replace("\\", "/") + "/.." # type: str
installerPath = os.path.join(versionBuildPath, "installer/index.html") # type: str
installerURL = modVersion.InstallerURL # type: str
filesPath = os.path.join(versionBuildPath, "files/index.html") # type: str
filesRelativePath = os.path.relpath(filesPath, Paths.DownloadsBuildPath) # type: str
filesURL = modVersion.FilesURL # type: str
sourcesPath = os.path.join(versionBuildPath, "sources/index.html") # type: str
sourcesURL = modVersion.SourcesURL # type: str
_WriteDownload(installerPath, modInstallerTemplate, basePath,
modName, str(modVersion.Version), _typeInstaller, str(modLatestVersion.GameVersion), modLatestVersion.ReleaseDate,
filesRelativePath, licensePath, installerURL)
_WriteDownload(filesPath, modFilesTemplate, basePath,
modName, str(modVersion.Version), _typeFiles, str(modVersion.GameVersion), modVersion.ReleaseDate,
filesRelativePath, licensePath, filesURL)
_WriteDownload(sourcesPath, sourcesTemplate, basePath,
modName, str(modVersion.Version), _typeSources, str(modVersion.GameVersion), modVersion.ReleaseDate,
filesRelativePath, licensePath, sourcesURL)
for modNamespace, modVersions in Distribution.Previews.items(): # type: str, typing.List[Distribution.ModVersion]
modBuildPath = os.path.join(Paths.DownloadsBuildPath, "Mods", modNamespace) # type: str
licensePath = os.path.join(modBuildPath.replace(Paths.DownloadsBuildPath + os.path.sep, ""), _licenseName) # type: str
licensePath = licensePath.replace("\\", "/")
with open(os.path.join(Paths.DownloadsSourcesPath, "mods", modNamespace + ".json")) as modInformationFile:
modInformation = decoder.JSONDecoder().decode(modInformationFile.read()) # type: dict
if not modInformation["Mod"]["Age"]:
modInstallerTemplate = downloadInstallerTemplate # type: str
modFilesTemplate = downloadFilesTemplate # type: str
else:
modInstallerTemplate = downloadInstallerAgeTemplate # type: str
modFilesTemplate = downloadFilesAgeTemplate # type: str
if not modInformation["Sources"]["Age"]:
sourcesTemplate = downloadSourcesTemplate # type: str
else:
sourcesTemplate = downloadSourcesAgeTemplate # type: str
mod = Mods.GetMod(modNamespace) # type: Mods.Mod
modName = mod.GetName() # type: typing.Optional[str]
for modVersion in modVersions: # type: Distribution.ModVersion
versionBuildPath = os.path.join(modBuildPath, str(modVersion.Version), modVersion.ConcealerFolderName)
basePath = os.path.relpath(Paths.DownloadsBuildPath, versionBuildPath).replace("\\", "/") + "/.." # type: str
installerPath = os.path.join(versionBuildPath, "installer/index.html") # type: str
installerURL = modVersion.InstallerURL # type: str
filesPath = os.path.join(versionBuildPath, "files/index.html") # type: str
filesRelativePath = os.path.relpath(filesPath, Paths.DownloadsBuildPath) # type: str
filesURL = modVersion.FilesURL # type: str
sourcesPath = os.path.join(versionBuildPath, "sources/index.html") # type: str
sourcesURL = modVersion.SourcesURL # type: str
_WriteDownload(installerPath, modInstallerTemplate, basePath,
modName, str(modVersion.Version), _typeFiles, str(modVersion.GameVersion), modVersion.ReleaseDate,
filesRelativePath, licensePath, installerURL)
_WriteDownload(filesPath, modFilesTemplate, basePath,
modName, str(modVersion.Version), _typeFiles, str(modVersion.GameVersion), modVersion.ReleaseDate,
filesRelativePath, licensePath, filesURL)
_WriteDownload(sourcesPath, sourcesTemplate, basePath,
modName, str(modVersion.Version), _typeSources, str(modVersion.GameVersion), modVersion.ReleaseDate,
filesRelativePath, licensePath, sourcesURL)
def __init__(self, applicationPath: str):
self.Name = os.path.split(applicationPath)[1] # type: str
self.PointerDirectoryPath = applicationPath # type: str
self.Special = "" # type: str
with open(os.path.join(self.PointerDirectoryPath,
"Application.json")) as applicationPointerFile:
applicationPointerDictionary = decoder.JSONDecoder().decode(
applicationPointerFile.read()
) # type: typing.Dict[str, typing.Any]
special = applicationPointerDictionary.get("Special") # type: str
if isinstance(special, str):
self.Special = special
self.ExecutablePath = None # type: typing.Union[str, None]
self.ExecutableSpecial = None # type: typing.Union[str, None]
if "Paths" in applicationPointerDictionary:
for pathDictionary in applicationPointerDictionary[
"Paths"]: # type: typing.Dict[str, str]
executableFullFilePath = None # type: typing.Optional[str]
executableRegistryKey = pathDictionary.get(
"Registry Key") # type: str
executableFilePath = pathDictionary.get(
"File Path") # type: str
executableSpecial = pathDictionary.get("Special") # type: str
registryKeyPath = None # type: typing.Optional[str]
if isinstance(executableRegistryKey, str):
executableRegistryKey = executableRegistryKey.replace(
"/", "\\")
if executableRegistryKey.count("") < 1:
continue
if platform.system() == "Windows":
try:
registryKeyPath = Registry.ReadRegistryFullKey(
executableRegistryKey)
except:
pass
if not isinstance(registryKeyPath,
str) and registryKeyPath is not None:
raise Exception(
"Application registry key value is not a string"
)
if registryKeyPath is not None:
if isinstance(executableFilePath, str):
executableFullFilePath = os.path.normpath(
os.path.join(registryKeyPath, executableFilePath))
else:
executableFullFilePath = os.path.normpath(
registryKeyPath)
pass
else:
if isinstance(executableFilePath, str):
executableFullFilePath = os.path.normpath(
os.path.join(self.PointerDirectoryPath,
os.path.normpath(executableFilePath)))
if executableFullFilePath is not None:
if os.path.exists(executableFullFilePath):
self.ExecutablePath = executableFullFilePath
self.ExecutableSpecial = executableSpecial
if self.ExecutablePath is None:
print("Cannot find valid path to application '" + self.Name +
"'.",
file=sys.stderr)
def BuildPackageChanges() -> bool:
if not Package.CanBuildPackage():
return False
for package in Mod.GetCurrentMod().Packages: # type: Mod.Package
baseFileExists = os.path.exists(
package.SourceBaseFilePath) # type: bool
loosePathExists = os.path.exists(package.SourceLoosePath) # type: bool
packageManifest = None # type: typing.Optional[typing.Dict[str, typing.Union[float, typing.Dict[str, float]]]]
if not os.path.exists(package.BuildFilePath):
_BuildPackageEverythingInternal(package)
return True
if os.path.exists(package.BuildManifestFilePath):
with open(package.BuildManifestFilePath) as packageManifestFile:
packageManifest = decoder.JSONDecoder().decode(
packageManifestFile.read())
if packageManifest is not None and not isinstance(
packageManifest, dict):
packageManifest = None
if packageManifest is not None and (not "Loose" in packageManifest
or not "Base" in packageManifest):
packageManifest = None
if packageManifest is None:
_BuildPackageEverythingInternal(package)
return True
else:
filesChanged = False # type: bool
if baseFileExists:
baseCurrentChangeTime = os.path.getmtime(
package.SourceBaseFilePath) # type: float
if packageManifest["Base"] != os.path.getmtime(
package.SourceBaseFilePath):
packageManifest["Base"] = baseCurrentChangeTime
filesChanged = True
if loosePathExists:
packageManifestLooseDictionary = packageManifest[
"Loose"] # type: dict
for entryFileName in list(
packageManifestLooseDictionary.keys()): # type: str
entryChangeTime = packageManifestLooseDictionary[
entryFileName] # type: float
entryFilePath = os.path.join(package.SourceLoosePath,
entryFileName) # type: str
if not os.path.exists(entryFilePath):
packageManifestLooseDictionary.pop(entryFileName)
filesChanged = True
continue
entryCurrentChangeTime = os.path.getmtime(
entryFilePath) # type: float
if entryCurrentChangeTime != entryChangeTime:
packageManifest["Loose"][
entryFileName] = entryCurrentChangeTime
filesChanged = True
for sourceDirectoryRoot, sourceDirectoryNames, sourceFileNames in os.walk(
package.SourceLoosePath
): # type: str, typing.List[str], typing.List[str]
for sourceFileName in sourceFileNames: # type: str
sourceFilePath = os.path.join(
sourceDirectoryRoot, sourceFileName) # type: str
relativeSourceFilePath = os.path.relpath(
sourceFilePath,
package.SourceLoosePath) # type: str
sourceFileDuplicate = False # type: bool
for entryFileName in packageManifest["Loose"].keys(
): # type: str
if entryFileName.lower(
) == relativeSourceFilePath.lower():
sourceFileDuplicate = True
break
if not sourceFileDuplicate:
packageManifest["Loose"][
relativeSourceFilePath] = os.path.getmtime(
sourceFilePath)
filesChanged = True
if filesChanged:
addingFilePaths = list() # type: typing.List[str]
if loosePathExists:
for sourceDirectoryRoot, sourceDirectoryNames, sourceFileNames in os.walk(
package.SourceLoosePath
): # type: str, typing.List[str], typing.List[str]
for sourceFileName in sourceFileNames: # type: str
# noinspection SpellCheckingInspection
if os.path.splitext(sourceFileName)[1].lower(
) == ".sourceinfo":
continue
sourceFilePath = os.path.join(
sourceDirectoryRoot,
sourceFileName) # type: str
if os.path.isfile(sourceFilePath):
addingFilePaths.append(sourceFilePath)
if baseFileExists:
Package.BuildPackage(
package.BuildFilePath,
baseFilePath=package.SourceBaseFilePath,
addingFilePaths=addingFilePaths)
else:
Package.BuildPackage(package.BuildFilePath,
addingFilePaths=addingFilePaths)
with open(package.BuildManifestFilePath,
"w+") as packageManifestFile:
packageManifestFile.write(
encoder.JSONEncoder(
indent="\t").encode(packageManifest))
return True
import socket
import traceback
from json import decoder
try:
from django.core import exceptions as django_exceptions
# Django JSON encoder uses the standard json encoder but can handle DateTime
from django.core.serializers import json as django_encoder
json_encoder = django_encoder.DjangoJSONEncoder()
except django_exceptions.ImproperlyConfigured:
from json import encoder
json_encoder = encoder.JSONEncoder()
json_decoder = decoder.JSONDecoder()
def customConvertJson(value):
"""\
Recursively process JSON values and do type conversions.
-change floats to ints
-change unicodes to strs
"""
if isinstance(value, float):
return int(value)
elif isinstance(value, unicode):
return str(value)
elif isinstance(value, list):
return [customConvertJson(item) for item in value]
elif isinstance(value, dict):
