def callOsfApi(provider='eartharxiv', filterDate='', verbose=True): # List to hold all our preprints preprints = [] # URLs for OSF API # API URL to list all preprint providers api_url_providers = "https://api.osf.io/v2/preprint_providers" # API URL to search/download preprints, NOTE: this is currently hard-coded to search EarthArXiv api_url_search = "https://api.osf.io/v2/preprints/?filter[provider]=" + provider # Are we filtering by date? if (filterDate != ''): api_url_search += '&filter[date_created][gte]=' + filterDate # Set up the headers to be sent as part of every API request # osf_token is unique to each user and needs to be obtained from OSF site, it's imported from api_token.py headers = { 'Content-Type': 'application/json', 'Authorization': 'Bearer {0}'.format(osf_token) } # Send a request to the search API, this example just asks for all preprints at EarthArXiv response = utils.queryAPI(api_url_search, headers) # Check the response status code, 200 indicates everything worked as expected if response.status_code == 200: # Extract the JSON data from the response json_object = utils.getJSON(response) # Total number of preprints in the results total_preprints = json_object['links']['meta']['total'] # Parse all the preprints in the response (the current 'page' of results) utils.parsePreprints(preprints, json_object, headers, verbose) # The API returns 10 preprints per "page". We need to look at the Links # data to see if there are additional pages. next = json_object['links']['next'] # Send a request to the search API, this time for the next page while (next != None): nextResponse = utils.queryAPI(next, headers) json_object = utils.getJSON(nextResponse) utils.parsePreprints(preprints, json_object, headers) next = json_object['links']['next'] else: # Something went wrong with the API call/response print("Error connecting to API, HTTP status code is: ", response.status_code) return preprints
def cmd_bitcoin(chat): data = utils.getJSON('https://www.bitmarket.pl/json/BTCPLN/ticker.json') if data is None: return txt = "1 BTC = " + str(data["last"]) + " PLN" data = utils.getJSON('https://www.bitmarket.pl/json/LTCPLN/ticker.json') if data is None: return skype.sendMessageToChat(chat, txt + "\n1 LTC = " + str(data["last"]) + " PLN")
def cmd_twitch(chat, nick): data = utils.getJSON("https://api.twitch.tv/kraken/streams/" + nick) if data is None: return if "error" in data: skype.sendMessageToChat(chat, "Nie ma takiego kanału.") return if data["stream"] is not None: nick = data["stream"]["channel"]["display_name"] game = data["stream"]["game"] status = data["stream"]["channel"]["status"] viewers = str(data["stream"]["viewers"]) fps = str(int(round(data["stream"]["average_fps"]))) if game: skype.sendMessageToChat( chat, nick + " streamuje " + game + " (" + status + ", " + viewers + " widzów, " + fps + " FPS)\nhttps://twitch.tv/" + nick) else: skype.sendMessageToChat( chat, nick + " prowadzi stream (" + status + ", " + viewers + " widzów, " + fps + " FPS)\nhttps://twitch.tv/" + nick) else: skype.sendMessageToChat(chat, nick + " obecnie nic nie streamuje!") return
def cmd_mtawiki ( chat, title ): data = utils.getJSON( "https://wiki.multitheftauto.com/api.php?action=query&list=search&format=json&prop=revisions&rvprop=content&srsearch=" + title ) if data is None: return if len(data["query"]["search"]) == 0: skype.sendMessageToChat( chat, "Nie ma takiej strony." ) return page = data["query"]["search"][0] txt = "https://wiki.mtasa.com/wiki/" + page["title"].replace(" ", "_") # Syntax data = utils.getJSON( "https://wiki.multitheftauto.com/api.php?format=json&action=query&prop=revisions&rvprop=content&titles=" + page["title"] ) if data is None: return for k, page in data["query"]["pages"].items(): content = page["revisions"][0]["*"] # check if this page is function or event and if is, grab syntax if content.find("function}}") != -1 or content.find("event}}") != -1: header_syntax_start = content.find("Syntax") if header_syntax_start == -1: header_syntax_start = content.find("Parameters") if header_syntax_start != -1: header_syntax_end = content.find("Returns", header_syntax_start) if header_syntax_end == -1: header_syntax_end = content.find("Example", header_syntax_start) if header_syntax_end == -1: header_syntax_end = len(content) syntax_startPos = content.find("<code>[lua", header_syntax_start, header_syntax_end) if syntax_startPos != -1: syntax_startPos = syntax_startPos + 10 syntax_startPos = content.find("]", syntax_startPos) + 1 syntax_endPos = content.find("</code>", syntax_startPos) syntax = content [ syntax_startPos : syntax_endPos ] syntax = syntax.replace("\n", "") syntax = syntax.replace(" ", "") txt = txt + "\n" + syntax else: # function without parameters txt = txt + "\nbrak parametrów" skype.sendMessageToChat( chat, txt )
def _loadTypes(self): # Get the data source data (i.e. the meta-metadata) locs = utils.getJSON(self.locsFile) # Get the schema for the locs file locsSchemaFile = os.path.join(self.top, locs['schemaDir'], locs['schema']) locsSchema = utils.getJSON(locsSchemaFile) # Validate the schema and locs #jsonschema.Draft4Validator.check_schema(locsSchema) #jsonschema.Draft4Validator(locsSchema, format_checker=jsonschema.FormatChecker()).validate(locs) # If no errors, then set the locs, along with the schemaDir and dataDir self.schemaDir = locs['schemaDir'] self.dataDir = locs['dataDir'] self.locations = locs['dataSources'] # Create the data source type hierarchy and set id and data hooks where appropriate self._root = TypeNode(None, "rootType") for dataSource in self.locations: dataTypeName = self.locations[dataSource]['dataType'] try: dataTypeNode = self._root.child(dataTypeName) except KeyError: dataTypeNode = self._root.add(dataTypeName) dataSourceNode = TypeNode(dataTypeNode, dataSource, \ _datahook=(lambda node, _id: self.getDataSourceInstance(node.name(), _id)), \ _idshook=(lambda node: self.getDataSourceIds(node.name()))) dataTypeNode.addNode(dataSourceNode) if 'hasIds' in self.locations[dataSource]: dataTypeNode.setIdsHook( (lambda node, child=dataSourceNode: child.getIds())) # Ensure that all top-level data types have an id source noIds = [ dataType for dataType in self._root.children() if dataType._idshook == None ] if len(noIds) != 0: raise AttributeError( "The following data types do not have id sources: {0}".format( [dataType.name() for dataType in noIds])) # Create an empty dictionary which will store the actual data from the data sources self.data = {}
def cmd_ets(chat): # servers status data = utils.getJSON('https://api.ets2mp.com/servers/') if data is None: return if data["error"] != "false": return txt = "Serwery TruckersMP:" total_players = 0 total_slots = 0 for server in data["response"]: txt = txt + "\n" + server["game"] + " " + server["name"] if server["online"]: txt = txt + " (" + str(server["players"]) + "/" + str( server["maxplayers"]) + ")" total_players = total_players + server["players"] total_slots = total_slots + server["maxplayers"] else: txt = txt + " (offline)" txt = txt + "\nŁącznie " + str(total_players) + "/" + str( total_slots) + " graczy." # game time data = utils.getJSON('https://api.ets2mp.com/game_time/') if data is not None: if data["error"]: return gameTime = datetime.datetime( 2015, 10, 25) + datetime.timedelta(minutes=data["game_time"]) txt = txt + "\nCzas w grze: " + gameTime.strftime('%H:%M') # song at TruckersFM data = utils.getURL('http://truckers.fm/') if data is not None: song_start = data.find( '<span id="currently_playing"> <span id="song"><span class="song-details">' ) + 73 if song_start != 72: song_end = data.find('</span>', song_start) txt = txt + "\nTruckersFM: " + data[song_start:song_end] skype.sendMessageToChat(chat, txt)
def cmd_twitchTop(chat, category=""): if category != "": data = utils.getJSON( "https://api.twitch.tv/kraken/search/games?type=suggest&live=1&limit=1&q=" + category) if data is None: return if len(data["games"]) == 0: skype.sendMessageToChat(chat, "Nie ma takiej kategorii.") return category = data["games"][0]["name"] data = utils.getJSON("https://api.twitch.tv/kraken/streams?limit=3&game=" + category) if data is None: return if len(data["streams"]) == 0: skype.sendMessageToChat(chat, "Brak streamów w kategorii " + category) return txt = "" for k, stream in enumerate(data["streams"]): nick = stream["channel"]["display_name"] status = stream["channel"]["status"] viewers = str(stream["viewers"]) fps = str(int(round(stream["average_fps"]))) if len(arg) > 0: txt = txt + "\n" + str( k + 1 ) + ". " + nick + " (" + status + ", " + viewers + " widzów, " + fps + " FPS) https://twitch.tv/" + nick else: game = stream["game"] txt = txt + "\n" + str( k + 1 ) + ". " + nick + " (" + game + ", " + status + ", " + viewers + " widzów, " + fps + " FPS) https://twitch.tv/" + nick if len(category) > 0: skype.sendMessageToChat( chat, "Top 3 streamów z kategorii " + category + ":" + txt) else: skype.sendMessageToChat(chat, "Top 3 streamów:" + txt) return
def loadSingleDataSource(self, dataSourceType): ''' Load the file containing data for this data source into a dictionary and place it in the top-level data variable. ''' dataSourceLocation = self.locations[dataSourceType] directory = self.getDataSourceDirectory(dataSourceType) filename = utils.makeJSONFilename(directory, os.path.basename(directory)) self.data[dataSourceType] = utils.getJSON(filename)
def checkFreeGamesSubreddit(): Timer(60 * 1, checkFreeGamesSubreddit).start() data = utils.getJSON( "https://www.reddit.com/r/freegamesonsteam/new.json?limit=1") if data is None or "data" not in data: return for link in data["data"]["children"]: if link["data"]["created_utc"] > utils.g_settings["lastFGTime"]: skype.sendMessageToChat( skype.g_chats["tr"], link["data"]["title"] + "\nhttps://reddit.com" + link["data"]["permalink"]) utils.g_settings["lastFGTime"] = link["data"]["created_utc"] utils.saveSettings()
def getAndPrintJSON(index, dir_fp, reverse=False, verbose=False): indent = 2 json_res_fp = os.path.join(dir_fp, str(index), 'result_after_merging.json') json_meta_fp = os.path.join(dir_fp, str(index), 'meta.json') meta = ut.getJSON(json_meta_fp) mid = meta['model_id'] mcat = meta['model_cat'] annoid = meta['anno_id'] if (not mcat in cats_to_load): return None if (verbose): print('{}\n{}\n{}'.format(index, mid, mcat)) res = ut.getJSON(json_res_fp) json_formatted_str = json.dumps(res, indent=indent) resl = json_formatted_str.splitlines() output = [] index = 0 for line in resl: if ('name' in line): level = int(line.split(':')[0].count(' ') / (2 * indent) - 1) name = line.split('"')[-2] name = name.replace('_', ' ') output.append({'name': name, 'level': level}) vocab.add(name) index = index + 1 if (output[0]['level'] > output[-1]['level']): output.reverse() output.insert(0, {'mid': mid}) output.insert(1, {'mcat': mcat}) output.insert(2, {'annoid': annoid}) return output
def otvetMailRu(q): #Запрос к mail.ru if (debug): logD("Обращение к ответам mail.ru") url = "https://go.mail.ru/answer_json?ajax_id=21&q" + urllib.parse.urlencode( [("", q)]) + "&num=2&sf=0&dwh_pagetype=search&dwh_platform=web" try: ans = utils.getJSON(url) res = (ans['results'][0]['banswer'] + "\n\n" + ans['results'][0]['answer']).replace("<b>", "").replace("</b>", "") if (res.count("!") + res.count(")") - res.count("(") + res.count("))") + res.count("!!") + res.count("!!!") > 4): return "" return res except: return ""
def cmd_steam(chat): data = utils.getJSON('http://store.steampowered.com/api/featured/') if data is None: return txt = "Polecane:\n" for game in data["large_capsules"]: price1 = str(game["final_price"])[:-2] if price1 == "": price1 = "0" price2 = str(game["final_price"])[-2:] txt = txt + game["name"] + " " + price1 + "." + price2 + "€" if game["discounted"] and game["discount_percent"] > 0: txt = txt + " [-" + str(game["discount_percent"]) + "%]" txt = txt + "\n" skype.sendMessageToChat(chat, txt)
def cmd_orct(chat): data = utils.getJSON("https://servers.openrct2.website/") if data is None or data["status"] != 200: skype.sendMessageToChat(chat, "Master server leży.") return total_players = 0 total_slots = 0 txt = "Serwery OpenRCT2:\n" for server in data["servers"]: if server["requiresPassword"]: txt = txt + " 🔒 " else: txt = txt + " " txt = txt + server["name"] + " " + str(server["players"]) + "/" + str( server["maxPlayers"]) + " (" + server["ip"]["v4"][0] + ":" + str( server["port"]) + ")\n" total_players = total_players + server["players"] total_slots = total_slots + server["maxPlayers"] skype.sendMessageToChat( chat, txt + "Łącznie " + str(total_players) + "/" + str(total_slots) + " graczy.")
def thequest(q): #Запрос к thequestion if (debug): logD("Обращение к html thequestion.ru") url = "https://thequestion.ru/search/questions?limit=2&offset=0&q" + urllib.parse.urlencode( [("", q)]) + "&sort=date" ans = utils.getJSON(url) try: nurl = ans['items'][0]['absoluteUrl'] time.sleep(0.84 + random.random() * 1) bts = urllib.request.urlopen(nurl) s = bts.read().decode('UTF-8') bts.close() startq = "class=\"answer__text\"><p>" endq = "</p></qml>" s = s[s.index(startq) + len(startq):] s = s[:s.index(endq)] s = s.replace(""", "\"").replace("<p>", "\n").replace("</p>", "").replace("<br>", "\n") return s except Exception as e: logD(e) return ""
def cmd_hitbox(chat, nick): print("cmd_hitbox " + nick) data = utils.getJSON("https://api.hitbox.tv/media/live/" + nick) if data is None: # todo: inspect this further skype.sendMessageToChat(chat, "api hitboxa jest zjebane") return if "error" in data: skype.sendMessageToChat(chat, "Nie ma takiego kanału.") return nick = data["livestream"][0]["media_display_name"] if data["livestream"][0]["media_is_live"] == "0": skype.sendMessageToChat(chat, nick + " obecnie nic nie streamuje!") return game = data["livestream"][0]["category_name"] status = data["livestream"][0]["media_status"] viewers = data["livestream"][0]["category_viewers"] skype.sendMessageToChat( chat, nick + " streamuje " + game + " (" + status + ", " + viewers + " widzów)" + "\nhttps://hitbox.tv/" + nick)
def checkGitHub(): print('checkGitHub') Timer(checkGithub_interval, checkGitHub).start() data = utils.getJSON( "https://api.github.com/repos/multitheftauto/mtasa-blue/commits?client_id=29c28b58cce0387e19a5&client_secret=62c2157f307108b637a8258a9f5e6ec549b69fbd&per_page=5" ) if data is None: return for commit in reversed(data): date = commit["commit"]["committer"]["date"] commitTime = time.mktime( datetime.datetime.strptime( date, "%Y-%m-%dT%H:%M:%SZ").timetuple()) # timestamp print( str(commitTime) + " > " + str(utils.g_settings["lastCommitTime"])) if commitTime > utils.g_settings["lastCommitTime"]: skype.sendMessageToChat( skype.g_chats["mta"], "[commit] " + commit["commit"]["message"] + " @ " + commit["author"]["login"] + "\n" + commit["html_url"]) utils.g_settings["lastCommitTime"] = commitTime utils.saveSettings()
return { 'id': _id, 'firstName': firstname, 'lastName': lastname, 'personType': personType } peopleMetadata = {} for person in people: _id = makeId(person) peopleMetadata[_id] = makePersonMetadata(_id, person.split()[0], person.split()[1], 'moderator') current = utils.getJSON("../data/people/metadata/metadata.json") utils.writeJSON(current, "../data/people/metadata/metadata_backup.json") alone = set([ _id for _id in current if current[_id]['personType'] == 'moderator' ]) - set(peopleMetadata.keys()) print(alone) print(len(peopleMetadata)) print(len(current)) current.update(peopleMetadata) print("-------") print(len(current)) utils.writeJSON(current, "../data/people/metadata/metadata.json") ## UNSAFE!!! ##
} peopleMetadata = {} for person in people: _id = makeId(person) peopleMetadata[_id] = makePersonMetadata( _id, person.split()[0], person.split()[1], 'candidate', data.debates[people[person]].debateMetadata.party) utils.writeJSON(peopleMetadata, "../data/people/metadata/metadata.json") ## UNSAFE!!! ## debateMetadata = utils.getJSON("../data/debates/metadata/metadata.json") utils.writeJSON(debateMetadata, "../data/debates/metadata/metadata_backup.json") for debateId in debateMetadata: debateMetadata[debateId]['participants'] = [ makeId(participant) for participant in participantsDict[debateId] ] utils.writeJSON(debateMetadata, "../data/debates/metadata/metadata.json") # ------------------------------------- VALIDATE ------------------------------------- # data.reset( ) # Reset the PDA, which resets the data manager, so the data sources that have changed get reloaded for year in [2000, 2004, 2008, 2012, 2016]: metadatas = sorted(
def getJSON(url): #TODO: Delete this stub return utils.getJSON(url)
def OnMessageStatus(message, status): if status == 'RECEIVED' or status == 'SENT': if (message.Chat.Name == skype.g_chats["mta"] or message.Chat.Name == skype.g_chats["test"]) and message.Body.find('#') != -1: bugID_start = message.Body.find('#') + 1 bugID_end = message.Body.find(' ', bugID_start) if bugID_end == -1: bugID_end = len(message.Body) bugID = message.Body[bugID_start:bugID_end] name, severity, status = notifications.checkMantisBug(bugID) if name is None: return skype.sendMessageToChat( message.Chat, "[" + severity + "/" + status + "] " + name + "\nhttps://bugs.mtasa.com/view.php?id=" + bugID) return if status == 'RECEIVED': for command in g_commands: #if message.Body.find(command.name) == 0: if message.Body.split()[0] == command.name: command.process(message.Body, message.Chat) return # other stuff if message.Body.find('v=') >= 0 or message.Body.find('youtu.be/') >= 0: link_start = message.Body.find('youtu.be/') + 9 if link_start - 9 == -1: link_start = message.Body.find('v=') + 2 link_end = message.Body.find('&', link_start) if link_end == -1: link_end = message.Body.find(' ', link_start) if link_end == -1: link_end = len(message.Body) vidID = message.Body[link_start:link_end] data = utils.getJSON( "https://www.googleapis.com/youtube/v3/videos?part=snippet,contentDetails&key=AIzaSyDViQNqCB7CxTiqS5YiBogXVBykLUtrUmY&id=" + vidID) if data is None: return if len(data["items"]) > 0: title = data["items"][0]["snippet"]["title"] skype.sendMessageToChat(message.Chat, 'YT: ' + title) return if message.Body.find('lenny') != -1: skype.sendMessageToChat(message.Chat, "( ͡° ͜ʖ ͡°)") return if message.Body.find( 'community.mtasa.com/index.php?p=resources&s=details&id=' ) >= 0: link_start = message.Body.find( 'community.mtasa.com/index.php?set_lang=eng&p=resources&s=details&id=' ) + 55 link_end = message.Body.find(' ', link_start) if link_end == -1: link_end = len(message.Body) if link_start >= 0: source = utils.getURL( "http://community.mtasa.com/index.php?p=resources&s=details&id=" + message.Body[link_start:link_end]) if source is None: return title_start = source.find('">rss</a></span>') + 16 title_end = source.find('</h2>', title_start) - 3 title = source[title_start:title_end] if title_start - 16 == -1: return author_start = source.find( '<tr><th>Author:</th><td><a href="?p=profile&id=') + 51 author_start = source.find('">', author_start) + 2 author_end = source.find('</a>', author_start) author = source[author_start:author_end] downloads_start = source.find( '<tr><th>Downloads:</th><td>') + 27 downloads_end = source.find('</td>', downloads_start) downloads = source[downloads_start:downloads_end] skype.sendMessageToChat( message.Chat, 'Community: ' + title + " @ " + author + " (" + downloads + " pobrań)") return if message.Body.find('store.steampowered.com/app/') >= 0: link_start = message.Body.find('store.steampowered.com/app/') + 27 link_end1 = message.Body.find('/', link_start) link_end2 = message.Body.find(' ', link_start) link_end = link_end1 if (link_end2 < link_end and link_end2 != -1) or link_end == -1: link_end = link_end2 if link_end == -1: link_end = len(message.Body) appID = message.Body[link_start:link_end] data = utils.getJSON( 'http://store.steampowered.com/api/appdetails?appids=' + appID) if data is None: return if not data[appID]["success"]: return data = data[appID]["data"] txt = "Steam: " + data["name"] if data["is_free"]: txt = txt + " (darmowe)" else: price1 = str(data["price_overview"]["final"])[:-2] if price1 == "": price1 = "0" price2 = str(data["price_overview"]["final"])[-2:] if data["price_overview"]["currency"] == "EUR": currency = "€" else: currency = "$" txt = txt + " " + price1 + "." + price2 + currency if data["price_overview"]["discount_percent"] > 0: txt = txt + " [-" + str( data["price_overview"]["discount_percent"]) + "%]" message.Chat.SendMessage(txt) return
meta_dir = '/media/starstorms/DATA/Insight/ShapeNet/stats/ShapeNetCore.v2' meta_fps = [] for dirName, subdirList, fileList in os.walk(meta_dir): for fname in fileList: fullpath = os.path.join(dirName, fname) meta_fps.append(fullpath) #%% Get all metadata and put into DF (takes a long time, use precomputed below) dfmeta = pd.DataFrame(columns=[ 'mid', 'cat', 'numv', 'xmin', 'xmax', 'centx', 'dx', 'ymin', 'ymax', 'centy', 'dy', 'zmin', 'zmax', 'centz', 'dz' ]) i = 0 for meta_fp in tqdm(meta_fps, total=len(meta_fps)): meta_js = ut.getJSON(meta_fp) mcat = meta_fp.split('/')[8] dfmeta.loc[i] = [ meta_js['id'], mcat, meta_js['numVertices'], meta_js['min'][0], meta_js['max'][0], meta_js['centroid'][0], meta_js['max'][0] - meta_js['min'][0], meta_js['min'][1], meta_js['max'][1], meta_js['centroid'][1], meta_js['max'][1] - meta_js['min'][1], meta_js['min'][2], meta_js['max'][2], meta_js['centroid'][2], meta_js['max'][2] - meta_js['min'][2] ] i = i + 1 #%% Write / reload the data from the previous cell # pd.DataFrame.to_csv(dfmeta, '/home/starstorms/Insight/ShapeNet/meta/df_meta_raw.csv') dfmeta = pd.read_csv('/home/starstorms/Insight/ShapeNet/meta/df_meta_raw.csv')