def fetch_author(a_id, api_key): utils.p('fetch author') try: a_url = paths.get_api_url() + 'accounts/' + a_id + '/' headers = utils.get_headers(api_key) r = rerequests.get(a_url, headers=headers) if r.status_code == 200: adata = r.json() if not hasattr(adata, 'id'): utils.p(adata) # utils.p(adata) tasks_queue.add_task((write_author, (a_id, adata))) if adata.get('gravatarHash') is not None: gravatar_path = paths.get_temp_dir( subdir='g/') + adata['gravatarHash'] + '.jpg' url = "https://www.gravatar.com/avatar/" + adata[ 'gravatarHash'] + '?d=404' r = rerequests.get(url, stream=False) if r.status_code == 200: with open(gravatar_path, 'wb') as f: f.write(r.content) adata['gravatarImg'] = gravatar_path elif r.status_code == '404': adata['gravatarHash'] = None utils.p('gravatar for author not available.') except Exception as e: utils.p(e) utils.p('finish fetch')
def get_assets_search(): bpy.app.debug_value = 2 results = [] preferences = bpy.context.preferences.addons['blenderkit'].preferences url = paths.get_api_url() + 'search/all' i = 0 while url is not None: headers = utils.get_headers(preferences.api_key) print('fetching assets from assets endpoint') print(url) retries = 0 while retries < 3: r = rerequests.get(url, headers=headers) try: adata = r.json() url = adata.get('next') print(i) i += 1 except Exception as e: print(e) print('failed to get next') if retries == 2: url = None if adata.get('results') != None: results.extend(adata['results']) retries = 3 print(f'fetched page {i}') retries += 1 fpath = assets_db_path() with open(fpath, 'w') as s: json.dump(results, s)
def fetch_categories(API_key, force=False): url = paths.get_api_url() + 'categories/' headers = utils.get_headers(API_key) tempdir = paths.get_temp_dir() categories_filepath = os.path.join(tempdir, 'categories.json') if os.path.exists(categories_filepath): catfile_age = time.time() - os.path.getmtime(categories_filepath) else: catfile_age = 10000000 # global catfetch_counter # catfetch_counter += 1 # bk_logger.debug('fetching categories: ', catfetch_counter) # bk_logger.debug('age of cat file', catfile_age) try: # read categories only once per day maximum, or when forced to do so. if catfile_age > 86400 or force: bk_logger.debug('requesting categories from server') r = rerequests.get(url, headers=headers) rdata = r.json() categories = rdata['results'] fix_category_counts(categories) # filter_categories(categories) #TODO this should filter categories for search, but not for upload. by now off. with open(categories_filepath, 'w', encoding='utf-8') as s: json.dump(categories, s, ensure_ascii=False, indent=4) tasks_queue.add_task((load_categories, ())) except Exception as e: bk_logger.debug('category fetching failed') bk_logger.exception(e) if not os.path.exists(categories_filepath): source_path = paths.get_addon_file(subpath='data' + os.sep + 'categories.json') shutil.copy(source_path, categories_filepath)
def get_rating(asset_id, headers): ''' Retrieve ratings from BlenderKit server. Can be run from a thread Parameters ---------- asset_id headers Returns ------- ratings - dict of type:value ratings ''' url = paths.get_api_url() + 'assets/' + asset_id + '/rating/' params = {} r = rerequests.get(url, params=params, verify=True, headers=headers) if r is None: return if r.status_code == 200: rj = r.json() ratings = {} # store ratings - send them to task queue for r in rj['results']: ratings[r['ratingType']] = r['score'] tasks_queue.add_task( (store_rating_local, (asset_id, r['ratingType'], r['score']))) # store_rating_local(asset_id, type = r['ratingType'], value = r['score']) if len(rj['results']) == 0: # store empty ratings too, so that server isn't checked repeatedly tasks_queue.add_task((store_rating_local_empty, (asset_id, )))
def request_profile(api_key): a_url = paths.get_api_url() + 'me/' headers = utils.get_headers(api_key) r = rerequests.get(a_url, headers=headers) adata = r.json() if adata.get('user') is None: utils.p(adata) utils.p('getting profile failed') return None return adata
def get_rating(asset_id): user_preferences = bpy.context.preferences.addons['blenderkit'].preferences api_key = user_preferences.api_key headers = utils.get_headers(api_key) rl = paths.get_api_url( ) + 'assets/' + asset['asset_data']['id'] + '/rating/' rtypes = ['quality', 'working_hours'] for rt in rtypes: params = {'rating_type': rt} r = rerequests.get(r1, params=data, verify=True, headers=headers) print(r.text)
def fetch_gravatar(adata): utils.p('fetch gravatar') if adata.get('gravatarHash') is not None: gravatar_path = paths.get_temp_dir(subdir='g/') + adata['gravatarHash'] + '.jpg' if os.path.exists(gravatar_path): tasks_queue.add_task((write_gravatar, (adata['id'], gravatar_path))) return; url = "https://www.gravatar.com/avatar/" + adata['gravatarHash'] + '?d=404' r = rerequests.get(url, stream=False) if r.status_code == 200: with open(gravatar_path, 'wb') as f: f.write(r.content) tasks_queue.add_task((write_gravatar, (adata['id'], gravatar_path))) elif r.status_code == '404': adata['gravatarHash'] = None utils.p('gravatar for author not available.')
def get_download_url(asset_data, scene_id, api_key, tcom=None): ''''retrieves the download url. The server checks if user can download the item.''' mt = time.time() headers = utils.get_headers(api_key) data = {'scene_uuid': scene_id} r = None try: r = rerequests.get(asset_data['download_url'], params=data, headers=headers) except Exception as e: print(e) if tcom is not None: tcom.error = True if r == None: if tcom is not None: tcom.report = 'Connection Error' tcom.error = True return 'Connection Error' if r.status_code < 400: data = r.json() url = data['filePath'] asset_data['url'] = url asset_data['file_name'] = paths.extract_filename_from_url(url) return True if r.status_code == 403: r = 'You need Full plan to get this item.' # r1 = 'All materials and brushes are available for free. Only users registered to Standard plan can use all models.' # tasks_queue.add_task((ui.add_report, (r1, 5, colors.RED))) if tcom is not None: tcom.report = r tcom.error = True elif r.status_code >= 500: utils.p(r.text) if tcom is not None: tcom.report = 'Server error' tcom.error = True return False
def fetch_categories(API_key): url = paths.get_api_url() + 'categories/' headers = utils.get_headers(API_key) tempdir = paths.get_temp_dir() categories_filepath = os.path.join(tempdir, 'categories.json') try: r = rerequests.get(url, headers=headers) rdata = r.json() categories = rdata['results'] fix_category_counts(categories) # filter_categories(categories) #TODO this should filter categories for search, but not for upload. by now off. with open(categories_filepath, 'w') as s: json.dump(categories, s, indent=4) tasks_queue.add_task((load_categories, ())) except Exception as e: utils.p('category fetching failed') utils.p(e) if not os.path.exists(categories_filepath): source_path = paths.get_addon_file(subpath='data' + os.sep + 'categories.json') shutil.copy(source_path, categories_filepath)
def run(self): maxthreads = 50 query = self.query params = self.params global reports t = time.time() mt('search thread started') tempdir = paths.get_temp_dir('%s_search' % query['asset_type']) json_filepath = os.path.join( tempdir, '%s_searchresult.json' % query['asset_type']) headers = utils.get_headers(params['api_key']) rdata = {} rdata['results'] = [] if params['get_next']: with open(json_filepath, 'r') as infile: try: origdata = json.load(infile) urlquery = origdata['next'] # rparameters = {} if urlquery == None: return except: # in case no search results found on drive we don't do next page loading. params['get_next'] = False if not params['get_next']: url = paths.get_api_url() + 'search/' urlquery = url # rparameters = query urlquery = self.query_to_url() try: utils.p(urlquery) r = rerequests.get(urlquery, headers=headers) # , params = rparameters) # print(r.url) reports = '' # utils.p(r.text) except requests.exceptions.RequestException as e: print(e) reports = e # props.report = e return mt('response is back ') try: rdata = r.json() except Exception as inst: reports = r.text print(inst) mt('data parsed ') # filter results here: # todo remove this in future nresults = [] for d in rdata.get('results', []): # TODO this code is for filtering brush types, should vanish after we implement filter in Elastic mode = None if query['asset_type'] == 'brush': for p in d['parameters']: if p['parameterType'] == 'mode': mode = p['value'] if query['asset_type'] != 'brush' or (query.get('mode') != None and query['mode']) == mode: nresults.append(d) rdata['results'] = nresults # print('number of results: ', len(rdata.get('results', []))) if self.stopped(): utils.p('stopping search : ' + str(query)) return mt('search finished') i = 0 thumb_small_urls = [] thumb_small_filepaths = [] thumb_full_urls = [] thumb_full_filepaths = [] # END OF PARSING for d in rdata.get('results', []): get_author(d) for f in d['files']: # TODO move validation of published assets to server, too manmy checks here. if f['fileType'] == 'thumbnail' and f[ 'fileThumbnail'] != None and f[ 'fileThumbnailLarge'] != None: if f['fileThumbnail'] == None: f['fileThumbnail'] = 'NONE' if f['fileThumbnailLarge'] == None: f['fileThumbnailLarge'] = 'NONE' thumb_small_urls.append(f['fileThumbnail']) thumb_full_urls.append(f['fileThumbnailLarge']) imgname = paths.extract_filename_from_url( f['fileThumbnail']) imgpath = os.path.join(tempdir, imgname) thumb_small_filepaths.append(imgpath) imgname = paths.extract_filename_from_url( f['fileThumbnailLarge']) imgpath = os.path.join(tempdir, imgname) thumb_full_filepaths.append(imgpath) sml_thbs = zip(thumb_small_filepaths, thumb_small_urls) full_thbs = zip(thumb_full_filepaths, thumb_full_urls) # we save here because a missing thumbnail check is in the previous loop # we can also prepend previous results. These have downloaded thumbnails already... if params['get_next']: rdata['results'][0:0] = origdata['results'] with open(json_filepath, 'w') as outfile: json.dump(rdata, outfile) killthreads_sml = [] for k in thumb_sml_download_threads.keys(): if k not in thumb_small_filepaths: killthreads_sml.append(k) # do actual killing here? killthreads_full = [] for k in thumb_full_download_threads.keys(): if k not in thumb_full_filepaths: killthreads_full.append(k) # do actual killing here? # TODO do the killing/ stopping here! remember threads might have finished inbetween! if self.stopped(): utils.p('stopping search : ' + str(query)) return # this loop handles downloading of small thumbnails for imgpath, url in sml_thbs: if imgpath not in thumb_sml_download_threads and not os.path.exists( imgpath): thread = ThumbDownloader(url, imgpath) # thread = threading.Thread(target=download_thumbnail, args=([url, imgpath]), # daemon=True) thread.start() thumb_sml_download_threads[imgpath] = thread # threads.append(thread) if len(thumb_sml_download_threads) > maxthreads: while len(thumb_sml_download_threads) > maxthreads: threads_copy = thumb_sml_download_threads.copy( ) # because for loop can erase some of the items. for tk, thread in threads_copy.items(): if not thread.is_alive(): thread.join() # utils.p(x) del (thumb_sml_download_threads[tk]) # utils.p('fetched thumbnail ', i) i += 1 if self.stopped(): utils.p('stopping search : ' + str(query)) return idx = 0 while len(thumb_sml_download_threads) > 0: threads_copy = thumb_sml_download_threads.copy( ) # because for loop can erase some of the items. for tk, thread in threads_copy.items(): if not thread.is_alive(): thread.join() del (thumb_sml_download_threads[tk]) i += 1 if self.stopped(): utils.p('stopping search : ' + str(query)) return # start downloading full thumbs in the end for imgpath, url in full_thbs: if imgpath not in thumb_full_download_threads and not os.path.exists( imgpath): thread = ThumbDownloader(url, imgpath) # thread = threading.Thread(target=download_thumbnail, args=([url, imgpath]), # daemon=True) thread.start() thumb_full_download_threads[imgpath] = thread mt('thumbnails finished')
def run(self): r = rerequests.get(self.url, stream=False) if r.status_code == 200: with open(self.path, 'wb') as f: f.write(r.content)
def run(self): maxthreads = 50 query = self.query params = self.params global reports t = time.time() mt('search thread started') tempdir = paths.get_temp_dir('%s_search' % query['asset_type']) # json_filepath = os.path.join(tempdir, '%s_searchresult.json' % query['asset_type']) headers = utils.get_headers(params['api_key']) rdata = {} rdata['results'] = [] if params['get_next']: urlquery = self.result['next'] if not params['get_next']: urlquery = self.query_to_url() try: utils.p(urlquery) r = rerequests.get(urlquery, headers=headers) # , params = rparameters) # print(r.url) reports = '' # utils.p(r.text) except requests.exceptions.RequestException as e: print(e) reports = e # props.report = e return mt('response is back ') try: rdata = r.json() except Exception as inst: reports = r.text print(inst) mt('data parsed ') if not rdata.get('results'): utils.pprint(rdata) # if the result was converted to json and didn't return results, # it means it's a server error that has a clear message. # That's why it gets processed in the update timer, where it can be passed in messages to user. self.result = rdata return # print('number of results: ', len(rdata.get('results', []))) if self.stopped(): utils.p('stopping search : ' + str(query)) return mt('search finished') i = 0 thumb_small_urls = [] thumb_small_filepaths = [] thumb_full_urls = [] thumb_full_filepaths = [] # END OF PARSING for d in rdata.get('results', []): get_author(d) for f in d['files']: # TODO move validation of published assets to server, too manmy checks here. if f['fileType'] == 'thumbnail' and f[ 'fileThumbnail'] != None and f[ 'fileThumbnailLarge'] != None: if f['fileThumbnail'] == None: f['fileThumbnail'] = 'NONE' if f['fileThumbnailLarge'] == None: f['fileThumbnailLarge'] = 'NONE' thumb_small_urls.append(f['fileThumbnail']) thumb_full_urls.append(f['fileThumbnailLarge']) imgname = paths.extract_filename_from_url( f['fileThumbnail']) imgpath = os.path.join(tempdir, imgname) thumb_small_filepaths.append(imgpath) imgname = paths.extract_filename_from_url( f['fileThumbnailLarge']) imgpath = os.path.join(tempdir, imgname) thumb_full_filepaths.append(imgpath) sml_thbs = zip(thumb_small_filepaths, thumb_small_urls) full_thbs = zip(thumb_full_filepaths, thumb_full_urls) # we save here because a missing thumbnail check is in the previous loop # we can also prepend previous results. These have downloaded thumbnails already... if params['get_next']: rdata['results'][0:0] = self.result['results'] self.result = rdata # with open(json_filepath, 'w') as outfile: # json.dump(rdata, outfile) killthreads_sml = [] for k in thumb_sml_download_threads.keys(): if k not in thumb_small_filepaths: killthreads_sml.append(k) # do actual killing here? killthreads_full = [] for k in thumb_full_download_threads.keys(): if k not in thumb_full_filepaths: killthreads_full.append(k) # do actual killing here? # TODO do the killing/ stopping here! remember threads might have finished inbetween! if self.stopped(): utils.p('stopping search : ' + str(query)) return # this loop handles downloading of small thumbnails for imgpath, url in sml_thbs: if imgpath not in thumb_sml_download_threads and not os.path.exists( imgpath): thread = ThumbDownloader(url, imgpath) # thread = threading.Thread(target=download_thumbnail, args=([url, imgpath]), # daemon=True) thread.start() thumb_sml_download_threads[imgpath] = thread # threads.append(thread) if len(thumb_sml_download_threads) > maxthreads: while len(thumb_sml_download_threads) > maxthreads: threads_copy = thumb_sml_download_threads.copy( ) # because for loop can erase some of the items. for tk, thread in threads_copy.items(): if not thread.is_alive(): thread.join() # utils.p(x) del (thumb_sml_download_threads[tk]) # utils.p('fetched thumbnail ', i) i += 1 if self.stopped(): utils.p('stopping search : ' + str(query)) return idx = 0 while len(thumb_sml_download_threads) > 0: threads_copy = thumb_sml_download_threads.copy( ) # because for loop can erase some of the items. for tk, thread in threads_copy.items(): if not thread.is_alive(): thread.join() del (thumb_sml_download_threads[tk]) i += 1 if self.stopped(): utils.p('stopping search : ' + str(query)) return # start downloading full thumbs in the end for imgpath, url in full_thbs: if imgpath not in thumb_full_download_threads and not os.path.exists( imgpath): thread = ThumbDownloader(url, imgpath) # thread = threading.Thread(target=download_thumbnail, args=([url, imgpath]), # daemon=True) thread.start() thumb_full_download_threads[imgpath] = thread mt('thumbnails finished')
def download_blenderkit_material(asset_ref): from blenderkit import paths, append_link, utils, version_checker, rerequests import requests def create_asset_data(rdata, asset_type): for r in rdata['results']: if r['assetType'] == asset_type and len(r['files']) > 0: furl = None tname = None allthumbs = [] durl, tname = None, None for f in r['files']: if f['fileType'] == 'thumbnail': tname = paths.extract_filename_from_url( f['fileThumbnailLarge']) small_tname = paths.extract_filename_from_url( f['fileThumbnail']) allthumbs.append( tname) # TODO just first thumb is used now. tdict = {} for i, t in enumerate(allthumbs): tdict['thumbnail_%i'] = t if f['fileType'] == 'blend': durl = f['downloadUrl'].split('?')[0] # fname = paths.extract_filename_from_url(f['filePath']) if durl and tname: tooltip = blenderkit.search.generate_tooltip(r) r['author']['id'] = str(r['author']['id']) asset_data = { 'thumbnail': tname, 'thumbnail_small': small_tname, # 'thumbnails':allthumbs, 'download_url': durl, 'id': r['id'], 'asset_base_id': r['assetBaseId'], 'name': r['name'], 'asset_type': r['assetType'], 'tooltip': tooltip, 'tags': r['tags'], 'can_download': r.get('canDownload', True), 'verification_status': r['verificationStatus'], 'author_id': r['author']['id'], # 'author': r['author']['firstName'] + ' ' + r['author']['lastName'] # 'description': r['description'], } asset_data['downloaded'] = 0 # parse extra params needed for blender here params = utils.params_to_dict(r['parameters']) if asset_type == 'model': if params.get('boundBoxMinX') != None: bbox = { 'bbox_min': (float(params['boundBoxMinX']), float(params['boundBoxMinY']), float(params['boundBoxMinZ'])), 'bbox_max': (float(params['boundBoxMaxX']), float(params['boundBoxMaxY']), float(params['boundBoxMaxZ'])) } else: bbox = { 'bbox_min': (-.5, -.5, 0), 'bbox_max': (.5, .5, 1) } asset_data.update(bbox) if asset_type == 'material': asset_data['texture_size_meters'] = params.get( 'textureSizeMeters', 1.0) asset_data.update(tdict) r.update(asset_data) # main asset_base_id_str, asset_type_str = asset_ref.split() asset_type = asset_type_str.split(':')[1] scene_id = blenderkit.download.get_scene_id() reqstr = '?query=%s+%s+order:_score' % (asset_base_id_str, asset_type_str) reqstr += '&addon_version=%s' % version_checker.get_addon_version() reqstr += '&scene_uuid=%s' % scene_id url = paths.get_api_url() + 'search/' + reqstr api_key = user_preferences = C.preferences.addons[ 'blenderkit'].preferences.api_key headers = utils.get_headers(api_key) r = rerequests.get(url, headers=headers) rdata = r.json() create_asset_data(rdata, asset_type) asset_data = rdata['results'][0] has_url = blenderkit.download.get_download_url(asset_data, scene_id, api_key) file_names = paths.get_download_filenames(asset_data) file_name = file_names[0] if not os.path.exists(file_name): with open(file_name, "wb") as f: print("Downloading %s" % file_name) response = requests.get(asset_data['url'], stream=True) total_length = response.headers.get('Content-Length') if total_length is None: # no content length header f.write(response.content) else: dl = 0 for data in response.iter_content(chunk_size=4096): dl += len(data) f.write(data) material = append_link.append_material(file_names[-1]) return material