def upload_files(filepath, upload_data, files): headers = utils.get_headers(upload_data['token']) version_id = upload_data['id'] for f in files: bg_blender.progress('uploading %s' % f['type']) upload_info = { 'assetId': version_id, 'fileType': f['type'], 'fileIndex': f['index'], 'originalFilename': os.path.basename(f['file_path']) } upload_create_url = paths.get_bkit_url() + 'uploads/' upload = requests.post(upload_create_url, json=upload_info, headers=headers, verify=True) upload = upload.json() # upheaders = { # "accept": "application/json", # "Authorization": "Bearer %s" % upload_data['token'], # "Content-Type": "multipart/form-data", # "Content-Disposition": 'form-data; name="file"; filename=%s' % f['file_path'] # # } chunk_size = 1024 * 256 # file gets uploaded here: uploaded = False # s3 upload is now the only option for a in range(0, 20): if not uploaded: try: upload_response = requests.put(upload['s3UploadUrl'], data=upload_in_chunks(f['file_path'], chunk_size, f['type']), stream=True, verify=True) if upload_response.status_code == 200: uploaded = True else: bg_blender.progress('Upload failed, retry.') except Exception as e: bg_blender.progress('Upload %s failed, retrying' % f['type']) time.sleep(1) # confirm single file upload to bkit server upload_done_url = paths.get_bkit_url() + 'uploads_s3/' + upload['id'] + '/upload-file/' upload_response = requests.post(upload_done_url, headers=headers, verify=True) bg_blender.progress('finished uploading') return {'FINISHED'}
def draw(self, context): # draw asset properties here layout = self.layout user_preferences = bpy.context.preferences.addons[ 'blenderkit'].preferences if user_preferences.login_attempt: draw_login_progress(layout) return if user_preferences.api_key != '': me = bpy.context.window_manager.get('bkit profile') if me is not None: me = me['user'] # user name layout.label(text='Me: %s %s' % (me['firstName'], me['lastName'])) # layout.label(text='Email: %s' % (me['email'])) # plan information if me.get('currentPlanName') is not None: pn = me['currentPlanName'] pcoll = icons.icon_collections["main"] if pn == 'Free': my_icon = pcoll['free'] else: my_icon = pcoll['full'] row = layout.row() row.label(text='My plan:') row.label(text='%s plan' % pn, icon_value=my_icon.icon_id) if pn == 'Free': layout.operator( "wm.url_open", text="Change plan", icon='URL' ).url = paths.get_bkit_url() + paths.BLENDERKIT_PLANS # storage statistics # if me.get('sumAssetFilesSize') is not None: # TODO remove this when production server has these too. # layout.label(text='My public assets: %i MiB' % (me['sumAssetFilesSize'])) # if me.get('sumPrivateAssetFilesSize') is not None: # layout.label(text='My private assets: %i MiB' % (me['sumPrivateAssetFilesSize'])) if me.get('remainingPrivateQuota') is not None: layout.label(text='My free storage: %i MiB' % (me['remainingPrivateQuota'])) layout.operator( "wm.url_open", text="See my uploads", icon='URL' ).url = paths.get_bkit_url() + paths.BLENDERKIT_USER_ASSETS
def fetch_categories(API_key): BLENDERKIT_API_MAIN = "https://www.blenderkit.com/api/v1/" url = paths.get_bkit_url() + 'categories/' headers = { "accept": "application/json", "Authorization": "Bearer %s" % API_key } tempdir = paths.get_temp_dir() categories_filepath = os.path.join(tempdir, 'categories.json') try: r = requests.get(url, headers=headers) rdata = r.json() categories = rdata['results'] fix_category_counts(categories) # filter_categories(categories) #TODO this should filter categories for search, but not for upload. by now off. with open(categories_filepath, 'w') as s: json.dump(categories, s, indent=4) except: print('category fetching failed') if not os.path.exists(categories_filepath): source_path = paths.get_addon_file(subpath='data' + os.sep + 'categories.json') shutil.copy(source_path, categories_filepath)
def refresh_token(api_key_refresh): authenticator = oauth.SimpleOAuthAuthenticator( server_url=paths.get_bkit_url(), client_id=CLIENT_ID, ports=PORTS) auth_token, refresh_token = authenticator.get_refreshed_token( api_key_refresh) if auth_token is not None and refresh_token is not None: tasks_queue.add_task((write_tokens, (auth_token, refresh_token)))
def login_thread(signup=False): r_url = paths.get_oauth_landing_url() url = paths.get_bkit_url() thread = threading.Thread(target=login, args=([signup, url, r_url]), daemon=True) thread.start()
def fetch_categories(API_key): BLENDERKIT_API_MAIN = "https://www.blenderkit.com/api/v1/" url = paths.get_bkit_url() + 'categories/' headers = { "accept": "application/json", "Authorization": "Bearer %s" % API_key } tempdir = paths.get_temp_dir() categories_filepath = os.path.join(tempdir, 'categories.json') try: r = requests.get(url, headers=headers) rdata = r.json() categories = rdata['results'] fix_category_counts(categories) # filter_categories(categories) #TODO this should filter categories for search, but not for upload. by now off. with open(categories_filepath, 'w') as s: json.dump(categories, s, indent=4) except: # print('category fetching failed') if not os.path.exists(categories_filepath): source_path = paths.get_addon_file(subpath='data' + os.sep + 'categories.json') shutil.copy(source_path, categories_filepath)
def draw(self, context): # draw asset properties here layout = self.layout user_preferences = bpy.context.preferences.addons[ 'blenderkit'].preferences if user_preferences.login_attempt: draw_login_progress(layout) return if user_preferences.enable_oauth: draw_login_buttons(layout) if user_preferences.api_key != '': me = bpy.context.window_manager.get('bkit profile') if me is not None: me = me['user'] layout.label(text='User: %s %s' % (me['firstName'], me['lastName'])) layout.label(text='Email: %s' % (me['email'])) if me.get( 'sumAssetFilesSize' ) is not None: # TODO remove this when production server has these too. layout.label(text='Public assets: %i MiB' % (me['sumAssetFilesSize'])) if me.get('sumPrivateAssetFilesSize') is not None: layout.label(text='Private assets: %i MiB' % (me['sumPrivateAssetFilesSize'])) if me.get('remainingPrivateQuota') is not None: layout.label(text='Remaining private storage: %i MiB' % (me['remainingPrivateQuota'])) layout.operator( "wm.url_open", text="See my uploads", icon='URL' ).url = paths.get_bkit_url() + paths.BLENDERKIT_USER_ASSETS
def upload_files(filepath, upload_data, files): headers = {"accept": "application/json", "Authorization": "Bearer %s" % upload_data['token']} version_id = upload_data['id'] for f in files: bg_blender.progress('uploading %s' % f['type']) upload_info = { 'assetId': version_id, 'fileType': f['type'], 'fileIndex': f['index'], 'originalFilename': os.path.basename(f['file_path']) } upload_create_url = paths.get_bkit_url() + 'uploads/' upload = requests.post(upload_create_url, json=upload_info, headers=headers, verify=True) upload = upload.json() upheaders = { "accept": "application/json", "Authorization": "Bearer %s" % upload_data['token'], "Content-Type": "multipart/form-data", "Content-Disposition": 'form-data; name="file"; filename=%s' % f['file_path'] } chunk_size = 1024 * 256 # file gets uploaded here: uploaded = False # s3 upload is now the only option for a in range(0, 20): if not uploaded: try: upload_response = requests.put(upload['s3UploadUrl'], data=upload_in_chunks(f['file_path'], chunk_size, f['type']), stream=True, verify=True) # print('upload response') # print(upload_response.text) uploaded = True except Exception as e: bg_blender.progress('Upload %s failed, retrying' % f['type']) time.sleep(1) # confirm single file upload to bkit server upload_done_url = paths.get_bkit_url() + 'uploads_s3/' + upload['id'] + '/upload-file/' upload_response = requests.post(upload_done_url, headers=headers, verify=True) bg_blender.progress('finished uploading') return {'FINISHED'}
def refresh_token_thread(): preferences = bpy.context.preferences.addons['blenderkit'].preferences if len(preferences.api_key_refresh) > 0: url = paths.get_bkit_url() thread = threading.Thread(target=refresh_token, args=([preferences.api_key_refresh, url]), daemon=True) thread.start()
def rerequest(method, url, recursion=0, **kwargs): # first get any additional args from kwargs immediate = False if kwargs.get('immediate'): immediate = kwargs['immediate'] kwargs.pop('immediate') # first normal attempt try: response = requests.request(method, url, **kwargs) except Exception as e: print(e) return None bk_logger.debug(url + str(kwargs)) bk_logger.debug(response.status_code) if response.status_code == 401: try: rdata = response.json() except: rdata = {} tasks_queue.add_task((ui.add_report, (method + ' request Failed.' + str(rdata.get('detail')),))) if rdata.get('detail') == 'Invalid token.': user_preferences = bpy.context.preferences.addons['blenderkit'].preferences if user_preferences.api_key != '': if user_preferences.enable_oauth and user_preferences.api_key_refresh != '': tasks_queue.add_task((ui.add_report, ( 'refreshing token. If this fails, please login in BlenderKit Login panel.', 10))) refresh_url = paths.get_bkit_url() auth_token, refresh_token, oauth_response = bkit_oauth.refresh_token( user_preferences.api_key_refresh, refresh_url) # bk_logger.debug(auth_token, refresh_token) if auth_token is not None: if immediate == True: # this can write tokens occasionally into prefs. used e.g. in upload. Only possible # in non-threaded tasks bpy.context.preferences.addons['blenderkit'].preferences.api_key = auth_token bpy.context.preferences.addons['blenderkit'].preferences.api_key_refresh = refresh_token else: tasks_queue.add_task((bkit_oauth.write_tokens, (auth_token, refresh_token, oauth_response))) kwargs['headers'] = utils.get_headers(auth_token) response = requests.request(method, url, **kwargs) bk_logger.debug('reresult', response.status_code) if response.status_code >= 400: bk_logger.debug('reresult', response.text) tasks_queue.add_task((ui.add_report, ( response.text, 10))) else: tasks_queue.add_task((ui.add_report, ( 'Refreshing token failed.Please login manually.', 10))) # tasks_queue.add_task((bkit_oauth.write_tokens, ('', '', ''))) tasks_queue.add_task((bpy.ops.wm.blenderkit_login, ('INVOKE_DEFAULT',)), fake_context=True) return response
def login(signup): r_url = paths.get_oauth_landing_url() authenticator = oauth.SimpleOAuthAuthenticator( server_url=paths.get_bkit_url(), client_id=CLIENT_ID, ports=PORTS) auth_token, refresh_token = authenticator.get_new_token(register=signup, redirect_url=r_url) utils.p('tokens retrieved') tasks_queue.add_task((write_tokens, (auth_token, refresh_token)))
def login_thread(signup=False): global active_authenticator r_url = paths.get_oauth_landing_url() url = paths.get_bkit_url() authenticator = oauth.SimpleOAuthAuthenticator(server_url=url, client_id=CLIENT_ID, ports=PORTS) #we store authenticator globally to be able to ping the server if connection fails. active_authenticator = authenticator thread = threading.Thread(target=login, args=([signup, url, r_url, authenticator]), daemon=True) thread.start()
def refresh_token_thread(): preferences = bpy.context.preferences.addons['blenderkit'].preferences if len(preferences.api_key_refresh) > 0 and preferences.refresh_in_progress == False: preferences.refresh_in_progress = True url = paths.get_bkit_url() thread = threading.Thread(target=refresh_token, args=([preferences.api_key_refresh, url]), daemon=True) thread.start() else: ui.add_report('Already Refreshing token, will be ready soon.')
def upload_rating(asset): user_preferences = bpy.context.preferences.addons['blenderkit'].preferences api_key = user_preferences.api_key headers = utils.get_headers(api_key) asset_data = asset['asset_data'] bkit_ratings = asset.bkit_ratings # print('rating asset', asset_data['name'], asset_data['asset_base_id']) url = paths.get_bkit_url() + 'assets/' + asset['asset_data']['id'] + '/rating/' ratings = [ ] if bkit_ratings.rating_quality > 0.1: ratings.append(('quality', bkit_ratings.rating_quality)) if bkit_ratings.rating_work_hours > 0.1: ratings.append(('working_hours', round(bkit_ratings.rating_work_hours, 1))) thread = threading.Thread(target=uplaod_rating_thread, args=(url, ratings, headers)) thread.start() url = paths.get_bkit_url() + 'assets/' + asset['asset_data']['id'] + '/review' reviews = { 'reviewText': bkit_ratings.rating_compliments, 'reviewTextProblems': bkit_ratings.rating_problems, } if not (bkit_ratings.rating_compliments == '' and bkit_ratings.rating_compliments == ''): thread = threading.Thread(target=uplaod_review_thread, args=(url, reviews, headers)) thread.start() # the info that the user rated an item is stored in the scene s = bpy.context.scene s['assets rated'] = s.get('assets rated', {}) if bkit_ratings.rating_quality > 0.1 and bkit_ratings.rating_work_hours > 0.1: s['assets rated'][asset['asset_data']['asset_base_id']] = True
def rerequest(method, url, **kwargs): # first get any additional args from kwargs immediate = False if kwargs.get('immediate'): immediate = kwargs['immediate'] kwargs.pop('immediate') # first normal attempt response = requests.request(method, url, **kwargs) utils.p(url) utils.p(response.status_code) if response.status_code == 401: try: rdata = response.json() except: rdata = {} tasks_queue.add_task( (ui.add_report, (method + ' request Failed.' + str(rdata.get('detail')), ))) if rdata.get('detail') == 'Invalid token.': user_preferences = bpy.context.preferences.addons[ 'blenderkit'].preferences if user_preferences.api_key != '': if user_preferences.enable_oauth and user_preferences.api_key_refresh != '': tasks_queue.add_task((ui.add_report, ( 'refreshing token. If this fails, please login in BlenderKit Login panel.', 10))) refresh_url = paths.get_bkit_url() auth_token, refresh_token, oauth_response = bkit_oauth.refresh_token( user_preferences.api_key_refresh, refresh_url) # utils.p(auth_token, refresh_token) if auth_token is not None: if immediate == True: # this can write tokens occasionally into prefs. used e.g. in upload. Only possible # in non-threaded tasks bpy.context.preferences.addons[ 'blenderkit'].preferences.api_key = auth_token bpy.context.preferences.addons[ 'blenderkit'].preferences.api_key_refresh = refresh_token kwargs['headers'] = utils.get_headers(auth_token) response = requests.request(method, url, **kwargs) utils.p('reresult', response.status_code) if response.status_code >= 400: utils.p('reresult', response.text) return response
def mark_for_validation(self, context, asset_type): props = utils.get_upload_props() props.upload_state = 'marking for validation' user_preferences = bpy.context.preferences.addons['blenderkit'].preferences upload_data = { "verificationStatus": "ready" } url = paths.get_bkit_url() + 'assets/' headers = {"accept": "application/json", "Authorization": "Bearer %s" % user_preferences.api_key} url += props.id + '/' try: r = requests.patch(url, json=upload_data, headers=headers, verify=True) # files = files, props.upload_state = 'marked for validation' except requests.exceptions.RequestException as e: props.upload_state = str(e) props.uploading = False return {'CANCELLED'} return {'FINISHED'}
"index": 0, "file_path": export_data["thumbnail_path"] }, { "type": "blend", "index": 0, "file_path": fpath }] bg_blender.progress('uploading') upload_files(fpath, upload_data, files) # mark on server as uploaded confirm_data = {"verificationStatus": "uploaded"} url = paths.get_bkit_url() + 'assets/' headers = { "accept": "application/json", "Authorization": "Bearer %s" % upload_data['token'] } url += upload_data["id"] + '/' r = requests.patch(url, json=confirm_data, headers=headers, verify=True) # files = files, bg_blender.progress('upload finished successfully') except Exception as e: print(e)
def report_usages(): '''report the usage of assets to the server.''' mt = time.time() user_preferences = bpy.context.preferences.addons['blenderkit'].preferences api_key = user_preferences.api_key sid = get_scene_id() headers = {"accept": "application/json", "Authorization": "Bearer %s" % api_key} url = paths.get_bkit_url() + paths.BLENDERKIT_REPORT_URL assets = {} asset_obs = [] scene = bpy.context.scene asset_usages = {} for ob in scene.collection.objects: if ob.get('asset_data') != None: asset_obs.append(ob) for ob in asset_obs: asset_data = ob['asset_data'] abid = asset_data['asset_base_id'] if assets.get(abid) is None: asset_usages[abid] = {'count': 1} assets[abid] = asset_data else: asset_usages[abid]['count'] += 1 # brushes for b in bpy.data.brushes: if b.get('asset_data') != None: abid = b['asset_data']['asset_base_id'] asset_usages[abid] = {'count': 1} assets[abid] = b['asset_data'] # materials for ob in scene.collection.objects: for ms in ob.material_slots: m = ms.material if m is not None and m.get('asset_data') is not None: abid = m['asset_data']['asset_base_id'] if assets.get(abid) is None: asset_usages[abid] = {'count': 1} assets[abid] = m['asset_data'] else: asset_usages[abid]['count'] += 1 assets_list = [] assets_reported = scene.get('assets reported', {}) new_assets_count = 0 for k in asset_usages.keys(): if k not in assets_reported.keys(): data = asset_usages[k] list_item = { 'asset': k, 'usageCount': data['count'], 'proximitySet': data.get('proximity', []) } assets_list.append(list_item) new_assets_count += 1 if k not in assets_reported.keys(): assets_reported[k] = True scene['assets reported'] = assets_reported if new_assets_count == 0: print('no new assets were added') return; usage_report = { 'scene': sid, 'reportType': 'save', 'assetusageSet': assets_list } au = scene.get('assets used', {}) ad = scene.get('assets deleted', {}) ak = assets.keys() for k in au.keys(): if k not in ak: ad[k] = au[k] else: if k in ad: ad.pop(k) # scene['assets used'] = {} for k in ak: # rewrite assets used. scene['assets used'][k] = assets[k] ###########check ratings herer too: scene['assets rated'] = scene.get('assets rated', {}) for k in assets.keys(): scene['assets rated'][k] = scene['assets rated'].get(k, False) thread = threading.Thread(target=utils.requests_post_thread, args=(url, usage_report, headers)) thread.start() # r = requests.post(url, headers=headers, json=usage_report) mt = time.time() - mt print('report generation: ', mt)
def run(self): maxthreads = 300 maximages = 50 query = self.query params = self.params global reports t = time.time() mt('search thread started') tempdir = paths.get_temp_dir('%s_search' % query['asset_type']) json_filepath = os.path.join(tempdir, '%s_searchresult.json' % query['asset_type']) if query['token'] != '': headers = { "accept": "application/json", "Authorization": "Bearer %s" % query['token'], # "Content-Type": "application/json", } else: headers = { "accept": "application/json", # "Content-Type": "application/json", } rdata = {} rdata['results'] = [] if params['get_next']: with open(json_filepath, 'r') as infile: try: origdata = json.load(infile) urlquery = origdata['next'] if urlquery == None: return; except: # in case no search results found on drive we don't do next page loading. params['get_next'] = False if not params['get_next']: # build a new request url = paths.get_bkit_url() + 'search/' nquery = { # 'tags': query['keywords'], 'asset_type': query['asset_type'], } if query.get('category'): nquery['category_subtree'] = query['category'] # build request manually # TODO use real queries requeststring = '?query=' + query['keywords'].lower() + '+' # for i, q in enumerate(nquery): requeststring += q + ':' + str(nquery[q]) if i < len(nquery) - 1: requeststring += '+' requeststring += '&addon_version=%s' % params['addon_version'] if params.get('scene_uuid') is not None: requeststring += '&scene_uuid=%s' % params['scene_uuid'] urlquery = url + requeststring try: # print(urlquery) r = requests.get(urlquery, headers=headers) reports = '' # print(r.text) except requests.exceptions.RequestException as e: print(e) reports = e # props.report = e return mt('response is back ') try: rdata = r.json() except Exception as inst: reports = r.text print(inst) mt('data parsed ') # filter results here: # todo remove this in future nresults = [] for d in rdata.get('results', []): # TODO this code is for filtering brush types, should vanish after we implement filter in Elastic mode = None if query['asset_type'] == 'brush': for p in d['parameters']: if p['parameterType'] == 'mode': mode = p['value'] if query['asset_type'] != 'brush' or ( query.get('brushType') != None and query['brushType']) == mode: nresults.append(d) rdata['results'] = nresults print('number of results: ', len(rdata.get('results', []))) if self.stopped(): print('stopping search : ' + query['keywords']) return mt('search finished') i = 0 thumb_small_urls = [] thumb_small_filepaths = [] thumb_full_urls = [] thumb_full_filepaths = [] # END OF PARSING for d in rdata.get('results', []): for f in d['files']: # TODO move validation of published assets to server, too manmy checks here. if f['fileType'] == 'thumbnail' and f['fileThumbnail'] != None and f['fileThumbnailLarge'] != None: if f['fileThumbnail'] == None: f['fileThumbnail'] = 'NONE' if f['fileThumbnailLarge'] == None: f['fileThumbnailLarge'] = 'NONE' thumb_small_urls.append(f['fileThumbnail']) thumb_full_urls.append(f['fileThumbnailLarge']) imgname = paths.extract_filename_from_url(f['fileThumbnail']) imgpath = os.path.join(tempdir, imgname) thumb_small_filepaths.append(imgpath) imgname = paths.extract_filename_from_url(f['fileThumbnailLarge']) imgpath = os.path.join(tempdir, imgname) thumb_full_filepaths.append(imgpath) sml_thbs = zip(thumb_small_filepaths, thumb_small_urls) full_thbs = zip(thumb_full_filepaths, thumb_full_urls) # we save here because a missing thumbnail check is in the previous loop # we can also prepend previous results. These have allready thumbnails downloaded... if params['get_next']: rdata['results'][0:0] = origdata['results'] with open(json_filepath, 'w') as outfile: json.dump(rdata, outfile) killthreads_sml = [] for k in thumb_sml_download_threads.keys(): if k not in thumb_small_filepaths: killthreads_sml.append(k) # do actual killing here? killthreads_full = [] for k in thumb_full_download_threads.keys(): if k not in thumb_full_filepaths: killthreads_full.append(k) # do actual killing here? # TODO do the killing/ stopping here! remember threads might have finished inbetween! if self.stopped(): print('stopping search : ' + query['keywords']) return # this loop handles downloading of small thumbnails for imgpath, url in sml_thbs: if imgpath not in thumb_sml_download_threads and not os.path.exists(imgpath): thread = ThumbDownloader(url, imgpath) # thread = threading.Thread(target=download_thumbnail, args=([url, imgpath]), # daemon=True) thread.start() thumb_sml_download_threads[imgpath] = thread # threads.append(thread) if len(thumb_sml_download_threads) > maxthreads: while len(thumb_sml_download_threads) > maxthreads: threads_copy = thumb_sml_download_threads.copy() # because for loop can erase some of the items. for tk, thread in threads_copy.items(): if not thread.is_alive(): thread.join() # print(x) del (thumb_sml_download_threads[tk]) print('fetched thumbnail ', i) i += 1 if self.stopped(): print('stopping search : ' + query['keywords']) return idx = 0 while len(thumb_sml_download_threads) > 0: threads_copy = thumb_sml_download_threads.copy() # because for loop can erase some of the items. for tk, thread in threads_copy.items(): if not thread.is_alive(): thread.join() del (thumb_sml_download_threads[tk]) i += 1 if self.stopped(): print('stopping search : ' + query['keywords']) return # start downloading full thumbs in the end for imgpath, url in full_thbs: if imgpath not in thumb_full_download_threads and not os.path.exists(imgpath): thread = ThumbDownloader(url, imgpath) # thread = threading.Thread(target=download_thumbnail, args=([url, imgpath]), # daemon=True) thread.start() thumb_full_download_threads[imgpath] = thread mt('thumbnails finished')
def start_upload(self, context, asset_type, as_new, metadata_only): props = utils.get_upload_props() location = get_upload_location(props) props.upload_state = 'preparing upload' # do this for fixing long tags in some upload cases props.tags = props.tags[:] props.name = props.name.strip() # TODO move this to separate function # check for missing metadata if asset_type == 'MODEL': get_missing_data_model(props) if asset_type == 'SCENE': get_missing_data_scene(props) elif asset_type == 'MATERIAL': get_missing_data_material(props) elif asset_type == 'BRUSH': get_missing_data_brush(props) if props.report != '': self.report({'ERROR_INVALID_INPUT'}, props.report) return {'CANCELLED'} if as_new: props.asset_base_id = '' props.id = '' export_data, upload_data, eval_path_computing, eval_path_state, eval_path, props = get_upload_data(self, context, asset_type) # utils.pprint(upload_data) upload_data['parameters'] = params_to_dict( upload_data['parameters']) # weird array conversion only for upload, not for tooltips. binary_path = bpy.app.binary_path script_path = os.path.dirname(os.path.realpath(__file__)) basename, ext = os.path.splitext(bpy.data.filepath) # if not basename: # basename = os.path.join(basename, "temp") if not ext: ext = ".blend" tempdir = tempfile.mkdtemp() source_filepath = os.path.join(tempdir, "export_blenderkit" + ext) clean_file_path = paths.get_clean_filepath() data = { 'clean_file_path': clean_file_path, 'source_filepath': source_filepath, 'temp_dir': tempdir, 'export_data': export_data, 'upload_data': upload_data, 'debug_value': bpy.app.debug_value, } datafile = os.path.join(tempdir, BLENDERKIT_EXPORT_DATA_FILE) # check if thumbnail exists: if not os.path.exists(export_data["thumbnail_path"]): props.upload_state = 'Thumbnail not found' props.uploading = False return {'CANCELLED'} # first upload metadata to server, so it can be saved inside the current file url = paths.get_bkit_url() + 'assets/' headers = {"accept": "application/json", "Authorization": "Bearer %s" % upload_data['token']} # upload_data['license'] = 'ovejajojo' json_metadata = upload_data # json.dumps(upload_data, ensure_ascii=False).encode('utf8') global reports if props.asset_base_id == '': try: r = requests.post(url, json=json_metadata, headers=headers, verify=True) # files = files, props.upload_state = 'uploaded metadata' except requests.exceptions.RequestException as e: print(e) props.upload_state = str(e) props.uploading = False return {'CANCELLED'} else: url += props.id + '/' try: if not metadata_only: json_metadata["verificationStatus"] = "uploading" r = requests.put(url, json=json_metadata, headers=headers, verify=True) # files = files, props.upload_state = 'uploaded metadata' # parse the reqest # print('uploaded metadata') # print(r.text) except requests.exceptions.RequestException as e: print(e) props.upload_state = str(e) props.uploading = False return {'CANCELLED'} # props.upload_state = 'step 1' if metadata_only: props.uploading = False return {'FINISHED'} try: rj = r.json() if props.asset_base_id == '': props.asset_base_id = rj['assetBaseId'] props.id = rj['id'] upload_data['assetBaseId'] = props.asset_base_id upload_data['id'] = props.id bpy.ops.wm.save_mainfile() # fa props.uploading = True # save a copy of actual scene but don't interfere with the users models bpy.ops.wm.save_as_mainfile(filepath=source_filepath, compress=False, copy=True) with open(datafile, 'w') as s: json.dump(data, s) proc = subprocess.Popen([ binary_path, "--background", "-noaudio", clean_file_path, "--python", os.path.join(script_path, "upload_bg.py"), "--", datafile # ,filepath, tempdir ], bufsize=5000, stdout=subprocess.PIPE, stdin=subprocess.PIPE) bg_blender.add_bg_process(eval_path_computing=eval_path_computing, eval_path_state=eval_path_state, eval_path=eval_path, process_type='UPLOAD', process=proc, location=location) except Exception as e: props.upload_state = str(e) props.uploading = False return {'CANCELLED'} return {'FINISHED'}
"index": 0, "file_path": export_data["thumbnail_path"] }, { "type": "blend", "index": 0, "file_path": fpath }] bg_blender.progress('uploading') upload_files(fpath, upload_data, files) # mark on server as uploaded confirm_data = { "verificationStatus": "uploaded" } url = paths.get_bkit_url() + 'assets/' headers = {"accept": "application/json", "Authorization": "Bearer %s" % upload_data['token']} url += upload_data["id"] + '/' r = requests.patch(url, json=confirm_data, headers=headers, verify=True) # files = files, bg_blender.progress('upload finished successfully') except Exception as e: print(e) bg_blender.progress(e) sys.exit(1)
def run(self): maxthreads = 300 maximages = 50 query = self.query params = self.params global reports t = time.time() mt('search thread started') tempdir = paths.get_temp_dir('%s_search' % query['asset_type']) json_filepath = os.path.join(tempdir, '%s_searchresult.json' % query['asset_type']) if query['token'] != '': headers = { "accept": "application/json", "Authorization": "Bearer %s" % query['token'], # "Content-Type": "application/json", } else: headers = { "accept": "application/json", # "Content-Type": "application/json", } rdata = {} rdata['results'] = [] if params['get_next']: with open(json_filepath, 'r') as infile: try: origdata = json.load(infile) urlquery = origdata['next'] if urlquery == None: return; except: # in case no search results found on drive we don't do next page loading. params['get_next'] = False if not params['get_next']: # build a new request url = paths.get_bkit_url() + 'search/' nquery = { # 'tags': query['keywords'], 'asset_type': query['asset_type'], } if query.get('category'): nquery['category_subtree'] = query['category'] # build request manually # TODO use real queries requeststring = '?query=' + query['keywords'].lower() + '+' # for i, q in enumerate(nquery): requeststring += q + ':' + str(nquery[q]) if i < len(nquery) - 1: requeststring += '+' requeststring += '&addon_version=%s' % params['addon_version'] if params.get('scene_uuid') is not None: requeststring += '&scene_uuid=%s' % params['scene_uuid'] urlquery = url + requeststring try: # print(urlquery) r = requests.get(urlquery, headers=headers) reports = '' # print(r.text) except requests.exceptions.RequestException as e: print(e) reports = e # props.report = e return mt('response is back ') try: rdata = r.json() except Exception as inst: reports = r.text print(inst) mt('data parsed ') # filter results here: # todo remove this in future nresults = [] for d in rdata.get('results', []): # TODO this code is for filtering brush types, should vanish after we implement filter in Elastic mode = None if query['asset_type'] == 'brush': for p in d['parameters']: if p['parameterType'] == 'mode': mode = p['value'] if query['asset_type'] != 'brush' or ( query.get('brushType') != None and query['brushType']) == mode: nresults.append(d) rdata['results'] = nresults # print('number of results: ', len(rdata.get('results', []))) if self.stopped(): print('stopping search : ' + query['keywords']) return mt('search finished') i = 0 thumb_small_urls = [] thumb_small_filepaths = [] thumb_full_urls = [] thumb_full_filepaths = [] # END OF PARSING for d in rdata.get('results', []): for f in d['files']: # TODO move validation of published assets to server, too manmy checks here. if f['fileType'] == 'thumbnail' and f['fileThumbnail'] != None and f['fileThumbnailLarge'] != None: if f['fileThumbnail'] == None: f['fileThumbnail'] = 'NONE' if f['fileThumbnailLarge'] == None: f['fileThumbnailLarge'] = 'NONE' thumb_small_urls.append(f['fileThumbnail']) thumb_full_urls.append(f['fileThumbnailLarge']) imgname = paths.extract_filename_from_url(f['fileThumbnail']) imgpath = os.path.join(tempdir, imgname) thumb_small_filepaths.append(imgpath) imgname = paths.extract_filename_from_url(f['fileThumbnailLarge']) imgpath = os.path.join(tempdir, imgname) thumb_full_filepaths.append(imgpath) sml_thbs = zip(thumb_small_filepaths, thumb_small_urls) full_thbs = zip(thumb_full_filepaths, thumb_full_urls) # we save here because a missing thumbnail check is in the previous loop # we can also prepend previous results. These have allready thumbnails downloaded... if params['get_next']: rdata['results'][0:0] = origdata['results'] with open(json_filepath, 'w') as outfile: json.dump(rdata, outfile) killthreads_sml = [] for k in thumb_sml_download_threads.keys(): if k not in thumb_small_filepaths: killthreads_sml.append(k) # do actual killing here? killthreads_full = [] for k in thumb_full_download_threads.keys(): if k not in thumb_full_filepaths: killthreads_full.append(k) # do actual killing here? # TODO do the killing/ stopping here! remember threads might have finished inbetween! if self.stopped(): print('stopping search : ' + query['keywords']) return # this loop handles downloading of small thumbnails for imgpath, url in sml_thbs: if imgpath not in thumb_sml_download_threads and not os.path.exists(imgpath): thread = ThumbDownloader(url, imgpath) # thread = threading.Thread(target=download_thumbnail, args=([url, imgpath]), # daemon=True) thread.start() thumb_sml_download_threads[imgpath] = thread # threads.append(thread) if len(thumb_sml_download_threads) > maxthreads: while len(thumb_sml_download_threads) > maxthreads: threads_copy = thumb_sml_download_threads.copy() # because for loop can erase some of the items. for tk, thread in threads_copy.items(): if not thread.is_alive(): thread.join() # print(x) del (thumb_sml_download_threads[tk]) # print('fetched thumbnail ', i) i += 1 if self.stopped(): print('stopping search : ' + query['keywords']) return idx = 0 while len(thumb_sml_download_threads) > 0: threads_copy = thumb_sml_download_threads.copy() # because for loop can erase some of the items. for tk, thread in threads_copy.items(): if not thread.is_alive(): thread.join() del (thumb_sml_download_threads[tk]) i += 1 if self.stopped(): print('stopping search : ' + query['keywords']) return # start downloading full thumbs in the end for imgpath, url in full_thbs: if imgpath not in thumb_full_download_threads and not os.path.exists(imgpath): thread = ThumbDownloader(url, imgpath) # thread = threading.Thread(target=download_thumbnail, args=([url, imgpath]), # daemon=True) thread.start() thumb_full_download_threads[imgpath] = thread mt('thumbnails finished')