def callOsfApi(provider='eartharxiv', filterDate='', verbose=True):
# List to hold all our preprints
preprints = []
# URLs for OSF API
# API URL to list all preprint providers
api_url_providers = "https://api.osf.io/v2/preprint_providers"
# API URL to search/download preprints, NOTE: this is currently hard-coded to search EarthArXiv
api_url_search = "https://api.osf.io/v2/preprints/?filter[provider]=" + provider
# Are we filtering by date?
if (filterDate != ''):
api_url_search += '&filter[date_created][gte]=' + filterDate
# Set up the headers to be sent as part of every API request
# osf_token is unique to each user and needs to be obtained from OSF site, it's imported from api_token.py
headers = {
'Content-Type': 'application/json',
'Authorization': 'Bearer {0}'.format(osf_token)
}
# Send a request to the search API, this example just asks for all preprints at EarthArXiv
response = utils.queryAPI(api_url_search, headers)
# Check the response status code, 200 indicates everything worked as expected
if response.status_code == 200:
# Extract the JSON data from the response
json_object = utils.getJSON(response)
# Total number of preprints in the results
total_preprints = json_object['links']['meta']['total']
# Parse all the preprints in the response (the current 'page' of results)
utils.parsePreprints(preprints, json_object, headers, verbose)
# The API returns 10 preprints per "page". We need to look at the Links
# data to see if there are additional pages.
next = json_object['links']['next']
# Send a request to the search API, this time for the next page
while (next != None):
nextResponse = utils.queryAPI(next, headers)
json_object = utils.getJSON(nextResponse)
utils.parsePreprints(preprints, json_object, headers)
next = json_object['links']['next']
else:
# Something went wrong with the API call/response
print("Error connecting to API, HTTP status code is: ",
response.status_code)
return preprints
def cmd_bitcoin(chat):
data = utils.getJSON('https://www.bitmarket.pl/json/BTCPLN/ticker.json')
if data is None:
return
txt = "1 BTC = " + str(data["last"]) + " PLN"
data = utils.getJSON('https://www.bitmarket.pl/json/LTCPLN/ticker.json')
if data is None:
return
skype.sendMessageToChat(chat,
txt + "\n1 LTC = " + str(data["last"]) + " PLN")
def cmd_twitch(chat, nick):
data = utils.getJSON("https://api.twitch.tv/kraken/streams/" + nick)
if data is None:
return
if "error" in data:
skype.sendMessageToChat(chat, "Nie ma takiego kanału.")
return
if data["stream"] is not None:
nick = data["stream"]["channel"]["display_name"]
game = data["stream"]["game"]
status = data["stream"]["channel"]["status"]
viewers = str(data["stream"]["viewers"])
fps = str(int(round(data["stream"]["average_fps"])))
if game:
skype.sendMessageToChat(
chat,
nick + " streamuje " + game + " (" + status + ", " + viewers +
" widzów, " + fps + " FPS)\nhttps://twitch.tv/" + nick)
else:
skype.sendMessageToChat(
chat, nick + " prowadzi stream (" + status + ", " + viewers +
" widzów, " + fps + " FPS)\nhttps://twitch.tv/" + nick)
else:
skype.sendMessageToChat(chat, nick + " obecnie nic nie streamuje!")
return
def cmd_mtawiki ( chat, title ):
data = utils.getJSON( "https://wiki.multitheftauto.com/api.php?action=query&list=search&format=json&prop=revisions&rvprop=content&srsearch=" + title )
if data is None:
return
if len(data["query"]["search"]) == 0:
skype.sendMessageToChat( chat, "Nie ma takiej strony." )
return
page = data["query"]["search"][0]
txt = "https://wiki.mtasa.com/wiki/" + page["title"].replace(" ", "_")
# Syntax
data = utils.getJSON( "https://wiki.multitheftauto.com/api.php?format=json&action=query&prop=revisions&rvprop=content&titles=" + page["title"] )
if data is None:
return
for k, page in data["query"]["pages"].items():
content = page["revisions"][0]["*"]
# check if this page is function or event and if is, grab syntax
if content.find("function}}") != -1 or content.find("event}}") != -1:
header_syntax_start = content.find("Syntax")
if header_syntax_start == -1:
header_syntax_start = content.find("Parameters")
if header_syntax_start != -1:
header_syntax_end = content.find("Returns", header_syntax_start)
if header_syntax_end == -1:
header_syntax_end = content.find("Example", header_syntax_start)
if header_syntax_end == -1:
header_syntax_end = len(content)
syntax_startPos = content.find("<code>[lua", header_syntax_start, header_syntax_end)
if syntax_startPos != -1:
syntax_startPos = syntax_startPos + 10
syntax_startPos = content.find("]", syntax_startPos) + 1
syntax_endPos = content.find("</code>", syntax_startPos)
syntax = content [ syntax_startPos : syntax_endPos ]
syntax = syntax.replace("\n", "")
syntax = syntax.replace(" ", "")
txt = txt + "\n" + syntax
else:
# function without parameters
txt = txt + "\nbrak parametrów"
skype.sendMessageToChat( chat, txt )
def _loadTypes(self):
# Get the data source data (i.e. the meta-metadata)
locs = utils.getJSON(self.locsFile)
# Get the schema for the locs file
locsSchemaFile = os.path.join(self.top, locs['schemaDir'],
locs['schema'])
locsSchema = utils.getJSON(locsSchemaFile)
# Validate the schema and locs
#jsonschema.Draft4Validator.check_schema(locsSchema)
#jsonschema.Draft4Validator(locsSchema, format_checker=jsonschema.FormatChecker()).validate(locs)
# If no errors, then set the locs, along with the schemaDir and dataDir
self.schemaDir = locs['schemaDir']
self.dataDir = locs['dataDir']
self.locations = locs['dataSources']
# Create the data source type hierarchy and set id and data hooks where appropriate
self._root = TypeNode(None, "rootType")
for dataSource in self.locations:
dataTypeName = self.locations[dataSource]['dataType']
try:
dataTypeNode = self._root.child(dataTypeName)
except KeyError:
dataTypeNode = self._root.add(dataTypeName)
dataSourceNode = TypeNode(dataTypeNode, dataSource, \
_datahook=(lambda node, _id: self.getDataSourceInstance(node.name(), _id)), \
_idshook=(lambda node: self.getDataSourceIds(node.name())))
dataTypeNode.addNode(dataSourceNode)
if 'hasIds' in self.locations[dataSource]:
dataTypeNode.setIdsHook(
(lambda node, child=dataSourceNode: child.getIds()))
# Ensure that all top-level data types have an id source
noIds = [
dataType for dataType in self._root.children()
if dataType._idshook == None
]
if len(noIds) != 0:
raise AttributeError(
"The following data types do not have id sources: {0}".format(
[dataType.name() for dataType in noIds]))
# Create an empty dictionary which will store the actual data from the data sources
self.data = {}
def cmd_ets(chat):
# servers status
data = utils.getJSON('https://api.ets2mp.com/servers/')
if data is None:
return
if data["error"] != "false":
return
txt = "Serwery TruckersMP:"
total_players = 0
total_slots = 0
for server in data["response"]:
txt = txt + "\n" + server["game"] + " " + server["name"]
if server["online"]:
txt = txt + " (" + str(server["players"]) + "/" + str(
server["maxplayers"]) + ")"
total_players = total_players + server["players"]
total_slots = total_slots + server["maxplayers"]
else:
txt = txt + " (offline)"
txt = txt + "\nŁącznie " + str(total_players) + "/" + str(
total_slots) + " graczy."
# game time
data = utils.getJSON('https://api.ets2mp.com/game_time/')
if data is not None:
if data["error"]:
return
gameTime = datetime.datetime(
2015, 10, 25) + datetime.timedelta(minutes=data["game_time"])
txt = txt + "\nCzas w grze: " + gameTime.strftime('%H:%M')
# song at TruckersFM
data = utils.getURL('http://truckers.fm/')
if data is not None:
song_start = data.find(
'<span id="currently_playing"> <span id="song"><span class="song-details">'
) + 73
if song_start != 72:
song_end = data.find('</span>', song_start)
txt = txt + "\nTruckersFM: " + data[song_start:song_end]
skype.sendMessageToChat(chat, txt)
def cmd_twitchTop(chat, category=""):
if category != "":
data = utils.getJSON(
"https://api.twitch.tv/kraken/search/games?type=suggest&live=1&limit=1&q="
+ category)
if data is None:
return
if len(data["games"]) == 0:
skype.sendMessageToChat(chat, "Nie ma takiej kategorii.")
return
category = data["games"][0]["name"]
data = utils.getJSON("https://api.twitch.tv/kraken/streams?limit=3&game=" +
category)
if data is None:
return
if len(data["streams"]) == 0:
skype.sendMessageToChat(chat, "Brak streamów w kategorii " + category)
return
txt = ""
for k, stream in enumerate(data["streams"]):
nick = stream["channel"]["display_name"]
status = stream["channel"]["status"]
viewers = str(stream["viewers"])
fps = str(int(round(stream["average_fps"])))
if len(arg) > 0:
txt = txt + "\n" + str(
k + 1
) + ". " + nick + " (" + status + ", " + viewers + " widzów, " + fps + " FPS) https://twitch.tv/" + nick
else:
game = stream["game"]
txt = txt + "\n" + str(
k + 1
) + ". " + nick + " (" + game + ", " + status + ", " + viewers + " widzów, " + fps + " FPS) https://twitch.tv/" + nick
if len(category) > 0:
skype.sendMessageToChat(
chat, "Top 3 streamów z kategorii " + category + ":" + txt)
else:
skype.sendMessageToChat(chat, "Top 3 streamów:" + txt)
return
def loadSingleDataSource(self, dataSourceType):
'''
Load the file containing data for this data source into a dictionary and
place it in the top-level data variable.
'''
dataSourceLocation = self.locations[dataSourceType]
directory = self.getDataSourceDirectory(dataSourceType)
filename = utils.makeJSONFilename(directory,
os.path.basename(directory))
self.data[dataSourceType] = utils.getJSON(filename)
def checkFreeGamesSubreddit():
Timer(60 * 1, checkFreeGamesSubreddit).start()
data = utils.getJSON(
"https://www.reddit.com/r/freegamesonsteam/new.json?limit=1")
if data is None or "data" not in data:
return
for link in data["data"]["children"]:
if link["data"]["created_utc"] > utils.g_settings["lastFGTime"]:
skype.sendMessageToChat(
skype.g_chats["tr"], link["data"]["title"] +
"\nhttps://reddit.com" + link["data"]["permalink"])
utils.g_settings["lastFGTime"] = link["data"]["created_utc"]
utils.saveSettings()
def getAndPrintJSON(index, dir_fp, reverse=False, verbose=False):
indent = 2
json_res_fp = os.path.join(dir_fp, str(index), 'result_after_merging.json')
json_meta_fp = os.path.join(dir_fp, str(index), 'meta.json')
meta = ut.getJSON(json_meta_fp)
mid = meta['model_id']
mcat = meta['model_cat']
annoid = meta['anno_id']
if (not mcat in cats_to_load): return None
if (verbose): print('{}\n{}\n{}'.format(index, mid, mcat))
res = ut.getJSON(json_res_fp)
json_formatted_str = json.dumps(res, indent=indent)
resl = json_formatted_str.splitlines()
output = []
index = 0
for line in resl:
if ('name' in line):
level = int(line.split(':')[0].count(' ') / (2 * indent) - 1)
name = line.split('"')[-2]
name = name.replace('_', ' ')
output.append({'name': name, 'level': level})
vocab.add(name)
index = index + 1
if (output[0]['level'] > output[-1]['level']):
output.reverse()
output.insert(0, {'mid': mid})
output.insert(1, {'mcat': mcat})
output.insert(2, {'annoid': annoid})
return output
def otvetMailRu(q): #Запрос к mail.ru
if (debug): logD("Обращение к ответам mail.ru")
url = "https://go.mail.ru/answer_json?ajax_id=21&q" + urllib.parse.urlencode(
[("", q)]) + "&num=2&sf=0&dwh_pagetype=search&dwh_platform=web"
try:
ans = utils.getJSON(url)
res = (ans['results'][0]['banswer'] + "\n\n" +
ans['results'][0]['answer']).replace("<b>",
"").replace("</b>", "")
if (res.count("!") + res.count(")") - res.count("(") +
res.count("))") + res.count("!!") + res.count("!!!") > 4):
return ""
return res
except:
return ""
def cmd_steam(chat):
data = utils.getJSON('http://store.steampowered.com/api/featured/')
if data is None:
return
txt = "Polecane:\n"
for game in data["large_capsules"]:
price1 = str(game["final_price"])[:-2]
if price1 == "":
price1 = "0"
price2 = str(game["final_price"])[-2:]
txt = txt + game["name"] + " " + price1 + "." + price2 + "€"
if game["discounted"] and game["discount_percent"] > 0:
txt = txt + " [-" + str(game["discount_percent"]) + "%]"
txt = txt + "\n"
skype.sendMessageToChat(chat, txt)
def cmd_orct(chat):
data = utils.getJSON("https://servers.openrct2.website/")
if data is None or data["status"] != 200:
skype.sendMessageToChat(chat, "Master server leży.")
return
total_players = 0
total_slots = 0
txt = "Serwery OpenRCT2:\n"
for server in data["servers"]:
if server["requiresPassword"]:
txt = txt + " 🔒 "
else:
txt = txt + " "
txt = txt + server["name"] + " " + str(server["players"]) + "/" + str(
server["maxPlayers"]) + " (" + server["ip"]["v4"][0] + ":" + str(
server["port"]) + ")\n"
total_players = total_players + server["players"]
total_slots = total_slots + server["maxPlayers"]
skype.sendMessageToChat(
chat, txt + "Łącznie " + str(total_players) + "/" + str(total_slots) +
" graczy.")
def thequest(q): #Запрос к thequestion
if (debug): logD("Обращение к html thequestion.ru")
url = "https://thequestion.ru/search/questions?limit=2&offset=0&q" + urllib.parse.urlencode(
[("", q)]) + "&sort=date"
ans = utils.getJSON(url)
try:
nurl = ans['items'][0]['absoluteUrl']
time.sleep(0.84 + random.random() * 1)
bts = urllib.request.urlopen(nurl)
s = bts.read().decode('UTF-8')
bts.close()
startq = "class=\"answer__text\"><p>"
endq = "</p></qml>"
s = s[s.index(startq) + len(startq):]
s = s[:s.index(endq)]
s = s.replace(""",
"\"").replace("<p>",
"\n").replace("</p>",
"").replace("<br>", "\n")
return s
except Exception as e:
logD(e)
return ""
def cmd_hitbox(chat, nick):
print("cmd_hitbox " + nick)
data = utils.getJSON("https://api.hitbox.tv/media/live/" + nick)
if data is None:
# todo: inspect this further
skype.sendMessageToChat(chat, "api hitboxa jest zjebane")
return
if "error" in data:
skype.sendMessageToChat(chat, "Nie ma takiego kanału.")
return
nick = data["livestream"][0]["media_display_name"]
if data["livestream"][0]["media_is_live"] == "0":
skype.sendMessageToChat(chat, nick + " obecnie nic nie streamuje!")
return
game = data["livestream"][0]["category_name"]
status = data["livestream"][0]["media_status"]
viewers = data["livestream"][0]["category_viewers"]
skype.sendMessageToChat(
chat, nick + " streamuje " + game + " (" + status + ", " + viewers +
" widzów)" + "\nhttps://hitbox.tv/" + nick)
def checkGitHub():
print('checkGitHub')
Timer(checkGithub_interval, checkGitHub).start()
data = utils.getJSON(
"https://api.github.com/repos/multitheftauto/mtasa-blue/commits?client_id=29c28b58cce0387e19a5&client_secret=62c2157f307108b637a8258a9f5e6ec549b69fbd&per_page=5"
)
if data is None:
return
for commit in reversed(data):
date = commit["commit"]["committer"]["date"]
commitTime = time.mktime(
datetime.datetime.strptime(
date, "%Y-%m-%dT%H:%M:%SZ").timetuple()) # timestamp
print(
str(commitTime) + " > " + str(utils.g_settings["lastCommitTime"]))
if commitTime > utils.g_settings["lastCommitTime"]:
skype.sendMessageToChat(
skype.g_chats["mta"],
"[commit] " + commit["commit"]["message"] + " @ " +
commit["author"]["login"] + "\n" + commit["html_url"])
utils.g_settings["lastCommitTime"] = commitTime
utils.saveSettings()
return {
'id': _id,
'firstName': firstname,
'lastName': lastname,
'personType': personType
}
peopleMetadata = {}
for person in people:
_id = makeId(person)
peopleMetadata[_id] = makePersonMetadata(_id,
person.split()[0],
person.split()[1], 'moderator')
current = utils.getJSON("../data/people/metadata/metadata.json")
utils.writeJSON(current, "../data/people/metadata/metadata_backup.json")
alone = set([
_id for _id in current if current[_id]['personType'] == 'moderator'
]) - set(peopleMetadata.keys())
print(alone)
print(len(peopleMetadata))
print(len(current))
current.update(peopleMetadata)
print("-------")
print(len(current))
utils.writeJSON(current, "../data/people/metadata/metadata.json")
## UNSAFE!!! ##
}
peopleMetadata = {}
for person in people:
_id = makeId(person)
peopleMetadata[_id] = makePersonMetadata(
_id,
person.split()[0],
person.split()[1], 'candidate',
data.debates[people[person]].debateMetadata.party)
utils.writeJSON(peopleMetadata, "../data/people/metadata/metadata.json")
## UNSAFE!!! ##
debateMetadata = utils.getJSON("../data/debates/metadata/metadata.json")
utils.writeJSON(debateMetadata,
"../data/debates/metadata/metadata_backup.json")
for debateId in debateMetadata:
debateMetadata[debateId]['participants'] = [
makeId(participant) for participant in participantsDict[debateId]
]
utils.writeJSON(debateMetadata, "../data/debates/metadata/metadata.json")
# ------------------------------------- VALIDATE ------------------------------------- #
data.reset(
) # Reset the PDA, which resets the data manager, so the data sources that have changed get reloaded
for year in [2000, 2004, 2008, 2012, 2016]:
metadatas = sorted(
def getJSON(url): #TODO: Delete this stub
return utils.getJSON(url)
def OnMessageStatus(message, status):
if status == 'RECEIVED' or status == 'SENT':
if (message.Chat.Name == skype.g_chats["mta"] or message.Chat.Name
== skype.g_chats["test"]) and message.Body.find('#') != -1:
bugID_start = message.Body.find('#') + 1
bugID_end = message.Body.find(' ', bugID_start)
if bugID_end == -1:
bugID_end = len(message.Body)
bugID = message.Body[bugID_start:bugID_end]
name, severity, status = notifications.checkMantisBug(bugID)
if name is None:
return
skype.sendMessageToChat(
message.Chat, "[" + severity + "/" + status + "] " + name +
"\nhttps://bugs.mtasa.com/view.php?id=" + bugID)
return
if status == 'RECEIVED':
for command in g_commands:
#if message.Body.find(command.name) == 0:
if message.Body.split()[0] == command.name:
command.process(message.Body, message.Chat)
return
# other stuff
if message.Body.find('v=') >= 0 or message.Body.find('youtu.be/') >= 0:
link_start = message.Body.find('youtu.be/') + 9
if link_start - 9 == -1:
link_start = message.Body.find('v=') + 2
link_end = message.Body.find('&', link_start)
if link_end == -1:
link_end = message.Body.find(' ', link_start)
if link_end == -1:
link_end = len(message.Body)
vidID = message.Body[link_start:link_end]
data = utils.getJSON(
"https://www.googleapis.com/youtube/v3/videos?part=snippet,contentDetails&key=AIzaSyDViQNqCB7CxTiqS5YiBogXVBykLUtrUmY&id="
+ vidID)
if data is None:
return
if len(data["items"]) > 0:
title = data["items"][0]["snippet"]["title"]
skype.sendMessageToChat(message.Chat, 'YT: ' + title)
return
if message.Body.find('lenny') != -1:
skype.sendMessageToChat(message.Chat, "( ͡° ͜ʖ ͡°)")
return
if message.Body.find(
'community.mtasa.com/index.php?p=resources&s=details&id='
) >= 0:
link_start = message.Body.find(
'community.mtasa.com/index.php?set_lang=eng&p=resources&s=details&id='
) + 55
link_end = message.Body.find(' ', link_start)
if link_end == -1:
link_end = len(message.Body)
if link_start >= 0:
source = utils.getURL(
"http://community.mtasa.com/index.php?p=resources&s=details&id="
+ message.Body[link_start:link_end])
if source is None:
return
title_start = source.find('">rss</a></span>') + 16
title_end = source.find('</h2>', title_start) - 3
title = source[title_start:title_end]
if title_start - 16 == -1:
return
author_start = source.find(
'<tr><th>Author:</th><td><a href="?p=profile&id=') + 51
author_start = source.find('">', author_start) + 2
author_end = source.find('</a>', author_start)
author = source[author_start:author_end]
downloads_start = source.find(
'<tr><th>Downloads:</th><td>') + 27
downloads_end = source.find('</td>', downloads_start)
downloads = source[downloads_start:downloads_end]
skype.sendMessageToChat(
message.Chat, 'Community: ' + title + " @ " + author +
" (" + downloads + " pobrań)")
return
if message.Body.find('store.steampowered.com/app/') >= 0:
link_start = message.Body.find('store.steampowered.com/app/') + 27
link_end1 = message.Body.find('/', link_start)
link_end2 = message.Body.find(' ', link_start)
link_end = link_end1
if (link_end2 < link_end and link_end2 != -1) or link_end == -1:
link_end = link_end2
if link_end == -1:
link_end = len(message.Body)
appID = message.Body[link_start:link_end]
data = utils.getJSON(
'http://store.steampowered.com/api/appdetails?appids=' + appID)
if data is None:
return
if not data[appID]["success"]:
return
data = data[appID]["data"]
txt = "Steam: " + data["name"]
if data["is_free"]:
txt = txt + " (darmowe)"
else:
price1 = str(data["price_overview"]["final"])[:-2]
if price1 == "":
price1 = "0"
price2 = str(data["price_overview"]["final"])[-2:]
if data["price_overview"]["currency"] == "EUR":
currency = "€"
else:
currency = "$"
txt = txt + " " + price1 + "." + price2 + currency
if data["price_overview"]["discount_percent"] > 0:
txt = txt + " [-" + str(
data["price_overview"]["discount_percent"]) + "%]"
message.Chat.SendMessage(txt)
return
meta_dir = '/media/starstorms/DATA/Insight/ShapeNet/stats/ShapeNetCore.v2'
meta_fps = []
for dirName, subdirList, fileList in os.walk(meta_dir):
for fname in fileList:
fullpath = os.path.join(dirName, fname)
meta_fps.append(fullpath)
#%% Get all metadata and put into DF (takes a long time, use precomputed below)
dfmeta = pd.DataFrame(columns=[
'mid', 'cat', 'numv', 'xmin', 'xmax', 'centx', 'dx', 'ymin', 'ymax',
'centy', 'dy', 'zmin', 'zmax', 'centz', 'dz'
])
i = 0
for meta_fp in tqdm(meta_fps, total=len(meta_fps)):
meta_js = ut.getJSON(meta_fp)
mcat = meta_fp.split('/')[8]
dfmeta.loc[i] = [
meta_js['id'], mcat, meta_js['numVertices'], meta_js['min'][0],
meta_js['max'][0], meta_js['centroid'][0],
meta_js['max'][0] - meta_js['min'][0], meta_js['min'][1],
meta_js['max'][1], meta_js['centroid'][1],
meta_js['max'][1] - meta_js['min'][1], meta_js['min'][2],
meta_js['max'][2], meta_js['centroid'][2],
meta_js['max'][2] - meta_js['min'][2]
]
i = i + 1
#%% Write / reload the data from the previous cell
# pd.DataFrame.to_csv(dfmeta, '/home/starstorms/Insight/ShapeNet/meta/df_meta_raw.csv')
dfmeta = pd.read_csv('/home/starstorms/Insight/ShapeNet/meta/df_meta_raw.csv')
