def fetch_author(a_id, api_key):
utils.p('fetch author')
try:
a_url = paths.get_api_url() + 'accounts/' + a_id + '/'
headers = utils.get_headers(api_key)
r = requests.get(a_url, headers=headers)
if r.status_code == 200:
adata = r.json()
if not hasattr(adata, 'id'):
utils.p(adata)
# utils.p(adata)
tasks_queue.add_task((write_author, (a_id, adata)))
if adata.get('gravatarHash') is not None:
gravatar_path = paths.get_temp_dir(
subdir=None) + adata['gravatarHash'] + '.jpg'
url = "https://www.gravatar.com/avatar/" + adata[
'gravatarHash'] + '?d=404'
r = requests.get(url, stream=False)
if r.status_code == 200:
with open(gravatar_path, 'wb') as f:
f.write(r.content)
adata['gravatarImg'] = gravatar_path
except Exception as e:
utils.p(e)
utils.p('finish fetch')
def update_ratings_work_hours(self, context):
user_preferences = bpy.context.preferences.addons['blenderkit'].preferences
api_key = user_preferences.api_key
headers = utils.get_headers(api_key)
if not (hasattr(self, 'rating_work_hours')):
# first option is for rating of assets that are from scene
asset = self.id_data
bkit_ratings = asset.bkit_ratings
asset_id = asset['asset_data']['id']
else:
# this part is for operator rating:
bkit_ratings = self
asset_id = self.asset_id
if bkit_ratings.rating_work_hours > 0.45:
url = paths.get_api_url() + f'assets/{asset_id}/rating/'
store_rating_local(asset_id,
type='working_hours',
value=bkit_ratings.rating_work_hours)
ratings = [('working_hours', round(bkit_ratings.rating_work_hours, 1))]
tasks_queue.add_task(
(send_rating_to_thread_work_hours, (url, ratings, headers)),
wait=2.5,
only_last=True)
def execute(self, context):
user_preferences = bpy.context.preferences.addons[
'blenderkit'].preferences
api_key = user_preferences.api_key
headers = utils.get_headers(api_key)
url = paths.get_api_url() + f'assets/{self.asset_id}/rating/'
rtgs = []
if self.rating_quality_ui == '':
self.rating_quality = 0
else:
self.rating_quality = int(self.rating_quality_ui)
if self.rating_quality > 0.1:
rtgs = (('quality', self.rating_quality), )
tasks_queue.add_task(
(send_rating_to_thread_quality, (url, rtgs, headers)),
wait=2.5,
only_last=True)
if self.rating_work_hours > 0.1:
rtgs = (('working_hours', round(self.rating_work_hours, 1)), )
tasks_queue.add_task(
(send_rating_to_thread_work_hours, (url, rtgs, headers)),
wait=2.5,
only_last=True)
return {'FINISHED'}
def download_file(asset_data):
# this is a simple non-threaded way to download files for background resolution genenration tool
file_name = paths.get_download_filenames(asset_data)[
0] # prefer global dir if possible.
if check_existing(asset_data):
# this sends the thread for processing, where another check should occur, since the file might be corrupted.
utils.p('not downloading, already in db')
return file_name
preferences = bpy.context.preferences.addons['blenderkit'].preferences
api_key = preferences.api_key
with open(file_name, "wb") as f:
print("Downloading %s" % file_name)
headers = utils.get_headers(api_key)
response = requests.get(asset_data['url'], stream=True)
total_length = response.headers.get('Content-Length')
if total_length is None: # no content length header
f.write(response.content)
else:
dl = 0
for data in response.iter_content(chunk_size=4096):
dl += len(data)
print(dl)
f.write(data)
return file_name
def fetch_categories(API_key, force=False):
url = paths.get_api_url() + 'categories/'
headers = utils.get_headers(API_key)
tempdir = paths.get_temp_dir()
categories_filepath = os.path.join(tempdir, 'categories.json')
if os.path.exists(categories_filepath):
catfile_age = time.time() - os.path.getmtime(categories_filepath)
else:
catfile_age = 10000000
# global catfetch_counter
# catfetch_counter += 1
# bk_logger.debug('fetching categories: ', catfetch_counter)
# bk_logger.debug('age of cat file', catfile_age)
try:
# read categories only once per day maximum, or when forced to do so.
if catfile_age > 86400 or force:
bk_logger.debug('requesting categories from server')
r = rerequests.get(url, headers=headers)
rdata = r.json()
categories = rdata['results']
fix_category_counts(categories)
# filter_categories(categories) #TODO this should filter categories for search, but not for upload. by now off.
with open(categories_filepath, 'w', encoding='utf-8') as s:
json.dump(categories, s, ensure_ascii=False, indent=4)
tasks_queue.add_task((load_categories, ()))
except Exception as e:
bk_logger.debug('category fetching failed')
bk_logger.exception(e)
if not os.path.exists(categories_filepath):
source_path = paths.get_addon_file(subpath='data' + os.sep +
'categories.json')
shutil.copy(source_path, categories_filepath)
def get_assets_search():
bpy.app.debug_value = 2
results = []
preferences = bpy.context.preferences.addons['blenderkit'].preferences
url = paths.get_api_url() + 'search/all'
i = 0
while url is not None:
headers = utils.get_headers(preferences.api_key)
print('fetching assets from assets endpoint')
print(url)
retries = 0
while retries < 3:
r = rerequests.get(url, headers=headers)
try:
adata = r.json()
url = adata.get('next')
print(i)
i += 1
except Exception as e:
print(e)
print('failed to get next')
if retries == 2:
url = None
if adata.get('results') != None:
results.extend(adata['results'])
retries = 3
print(f'fetched page {i}')
retries += 1
fpath = assets_db_path()
with open(fpath, 'w') as s:
json.dump(results, s)
def run(self):
'''try to download file from blenderkit'''
asset_data = self.asset_data
tcom = self.tcom
scene_id = self.scene_id
api_key = self.api_key
# TODO get real link here...
has_url = get_download_url(asset_data, scene_id, api_key, tcom=tcom)
if not has_url:
tasks_queue.add_task(
(ui.add_report,
('Failed to obtain download URL for %s.' % asset_data['name'],
5, colors.RED)))
return
if tcom.error:
return
# only now we can check if the file already exists. This should have 2 levels, for materials and for brushes
# different than for the non free content. delete is here when called after failed append tries.
if check_existing(asset_data) and not tcom.passargs.get('delete'):
# this sends the thread for processing, where another check should occur, since the file might be corrupted.
tcom.downloaded = 100
utils.p('not downloading, trying to append again')
return
file_name = paths.get_download_filenames(asset_data)[
0] # prefer global dir if possible.
# for k in asset_data:
# print(asset_data[k])
if self.stopped():
utils.p('stopping download: ' + asset_data['name'])
return
with open(file_name, "wb") as f:
print("Downloading %s" % file_name)
headers = utils.get_headers(api_key)
response = requests.get(asset_data['url'], stream=True)
total_length = response.headers.get('Content-Length')
if total_length is None: # no content length header
f.write(response.content)
else:
tcom.file_size = int(total_length)
dl = 0
totdata = []
for data in response.iter_content(
chunk_size=4096 *
32): #crashed here... why? investigate:
dl += len(data)
tcom.downloaded = dl
tcom.progress = int(100 * tcom.downloaded / tcom.file_size)
f.write(data)
if self.stopped():
utils.p('stopping download: ' + asset_data['name'])
os.remove(file_name)
return
def rerequest(method, url, recursion=0, **kwargs):
# first get any additional args from kwargs
immediate = False
if kwargs.get('immediate'):
immediate = kwargs['immediate']
kwargs.pop('immediate')
# first normal attempt
try:
response = requests.request(method, url, **kwargs)
except Exception as e:
print(e)
return None
bk_logger.debug(url + str(kwargs))
bk_logger.debug(response.status_code)
if response.status_code == 401:
try:
rdata = response.json()
except:
rdata = {}
tasks_queue.add_task((ui.add_report, (method + ' request Failed.' + str(rdata.get('detail')),)))
if rdata.get('detail') == 'Invalid token.':
user_preferences = bpy.context.preferences.addons['blenderkit'].preferences
if user_preferences.api_key != '':
if user_preferences.enable_oauth and user_preferences.api_key_refresh != '':
tasks_queue.add_task((ui.add_report, (
'refreshing token. If this fails, please login in BlenderKit Login panel.', 10)))
refresh_url = paths.get_bkit_url()
auth_token, refresh_token, oauth_response = bkit_oauth.refresh_token(
user_preferences.api_key_refresh, refresh_url)
# bk_logger.debug(auth_token, refresh_token)
if auth_token is not None:
if immediate == True:
# this can write tokens occasionally into prefs. used e.g. in upload. Only possible
# in non-threaded tasks
bpy.context.preferences.addons['blenderkit'].preferences.api_key = auth_token
bpy.context.preferences.addons['blenderkit'].preferences.api_key_refresh = refresh_token
else:
tasks_queue.add_task((bkit_oauth.write_tokens, (auth_token, refresh_token, oauth_response)))
kwargs['headers'] = utils.get_headers(auth_token)
response = requests.request(method, url, **kwargs)
bk_logger.debug('reresult', response.status_code)
if response.status_code >= 400:
bk_logger.debug('reresult', response.text)
tasks_queue.add_task((ui.add_report, (
response.text, 10)))
else:
tasks_queue.add_task((ui.add_report, (
'Refreshing token failed.Please login manually.', 10)))
# tasks_queue.add_task((bkit_oauth.write_tokens, ('', '', '')))
tasks_queue.add_task((bpy.ops.wm.blenderkit_login, ('INVOKE_DEFAULT',)), fake_context=True)
return response
def request_profile(api_key):
a_url = paths.get_api_url() + 'me/'
headers = utils.get_headers(api_key)
r = requests.get(a_url, headers=headers)
adata = r.json()
if adata.get('user') is None:
utils.p(adata)
utils.p('getting profile failed')
return None
return adata
def patch_individual_metadata(asset_id, metadata_dict, api_key):
upload_data = metadata_dict
url = paths.get_api_url() + 'assets/' + str(asset_id) + '/'
headers = utils.get_headers(api_key)
try:
r = rerequests.patch(url, json=upload_data, headers=headers, verify=True) # files = files,
except requests.exceptions.RequestException as e:
print(e)
return {'CANCELLED'}
return {'FINISHED'}
def get_rating(asset_id):
user_preferences = bpy.context.preferences.addons['blenderkit'].preferences
api_key = user_preferences.api_key
headers = utils.get_headers(api_key)
rl = paths.get_api_url(
) + 'assets/' + asset['asset_data']['id'] + '/rating/'
rtypes = ['quality', 'working_hours']
for rt in rtypes:
params = {'rating_type': rt}
r = rerequests.get(r1, params=data, verify=True, headers=headers)
print(r.text)
def upload_file(upload_data, f):
headers = utils.get_headers(upload_data['token'])
version_id = upload_data['id']
bg_blender.progress(
f"uploading {f['type']} {os.path.basename(f['file_path'])}")
upload_info = {
'assetId': version_id,
'fileType': f['type'],
'fileIndex': f['index'],
'originalFilename': os.path.basename(f['file_path'])
}
upload_create_url = paths.get_api_url() + 'uploads/'
upload = rerequests.post(upload_create_url,
json=upload_info,
headers=headers,
verify=True)
upload = upload.json()
#
chunk_size = 1024 * 1024 * 2
# utils.pprint(upload)
# file gets uploaded here:
uploaded = False
# s3 upload is now the only option
for a in range(0, 5):
if not uploaded:
try:
upload_response = requests.put(upload['s3UploadUrl'],
data=upload_in_chunks(
f['file_path'], chunk_size,
f['type']),
stream=True,
verify=True)
if 250 > upload_response.status_code > 199:
uploaded = True
else:
print(upload_response.text)
bg_blender.progress(f'Upload failed, retry. {a}')
except Exception as e:
print(e)
bg_blender.progress('Upload %s failed, retrying' % f['type'])
time.sleep(1)
# confirm single file upload to bkit server
print(upload)
upload_done_url = paths.get_api_url(
) + 'uploads_s3/' + upload['id'] + '/upload-file/'
upload_response = rerequests.post(upload_done_url,
headers=headers,
verify=True)
bg_blender.progress('finished uploading')
return uploaded
def verification_status_change_thread(asset_id, state, api_key):
upload_data = {
"verificationStatus": state
}
url = paths.get_api_url() + 'assets/' + str(asset_id) + '/'
headers = utils.get_headers(api_key)
try:
r = rerequests.patch(url, json=upload_data, headers=headers, verify=True) # files = files,
except requests.exceptions.RequestException as e:
print(e)
return {'CANCELLED'}
return {'FINISHED'}
def fetch_author(a_id, api_key):
utils.p('fetch author')
try:
a_url = paths.get_api_url() + 'accounts/' + a_id + '/'
headers = utils.get_headers(api_key)
r = requests.get(a_url, headers=headers)
adata = r.json()
if not hasattr(adata, 'id'):
utils.p(adata)
# utils.p(adata)
tasks_queue.add_task((write_author, (a_id, adata)))
except Exception as e:
utils.p(e)
utils.p('finish fetch')
def rerequest(method, url, **kwargs):
# first get any additional args from kwargs
immediate = False
if kwargs.get('immediate'):
immediate = kwargs['immediate']
kwargs.pop('immediate')
# first normal attempt
response = requests.request(method, url, **kwargs)
utils.p(url)
utils.p(response.status_code)
if response.status_code == 401:
try:
rdata = response.json()
except:
rdata = {}
tasks_queue.add_task(
(ui.add_report,
(method + ' request Failed.' + str(rdata.get('detail')), )))
if rdata.get('detail') == 'Invalid token.':
user_preferences = bpy.context.preferences.addons[
'blenderkit'].preferences
if user_preferences.api_key != '':
if user_preferences.enable_oauth and user_preferences.api_key_refresh != '':
tasks_queue.add_task((ui.add_report, (
'refreshing token. If this fails, please login in BlenderKit Login panel.',
10)))
refresh_url = paths.get_bkit_url()
auth_token, refresh_token, oauth_response = bkit_oauth.refresh_token(
user_preferences.api_key_refresh, refresh_url)
# utils.p(auth_token, refresh_token)
if auth_token is not None:
if immediate == True:
# this can write tokens occasionally into prefs. used e.g. in upload. Only possible
# in non-threaded tasks
bpy.context.preferences.addons[
'blenderkit'].preferences.api_key = auth_token
bpy.context.preferences.addons[
'blenderkit'].preferences.api_key_refresh = refresh_token
kwargs['headers'] = utils.get_headers(auth_token)
response = requests.request(method, url, **kwargs)
utils.p('reresult', response.status_code)
if response.status_code >= 400:
utils.p('reresult', response.text)
return response
def upload_files(filepath, upload_data, files):
headers = utils.get_headers(upload_data['token'])
version_id = upload_data['id']
for f in files:
bg_blender.progress('uploading %s' % f['type'])
upload_info = {
'assetId': version_id,
'fileType': f['type'],
'fileIndex': f['index'],
'originalFilename': os.path.basename(f['file_path'])
}
upload_create_url = paths.get_bkit_url() + 'uploads/'
upload = requests.post(upload_create_url, json=upload_info, headers=headers, verify=True)
upload = upload.json()
# upheaders = {
# "accept": "application/json",
# "Authorization": "Bearer %s" % upload_data['token'],
# "Content-Type": "multipart/form-data",
# "Content-Disposition": 'form-data; name="file"; filename=%s' % f['file_path']
#
# }
chunk_size = 1024 * 256
# file gets uploaded here:
uploaded = False
# s3 upload is now the only option
for a in range(0, 20):
if not uploaded:
try:
upload_response = requests.put(upload['s3UploadUrl'],
data=upload_in_chunks(f['file_path'], chunk_size, f['type']),
stream=True, verify=True)
if upload_response.status_code == 200:
uploaded = True
else:
bg_blender.progress('Upload failed, retry.')
except Exception as e:
bg_blender.progress('Upload %s failed, retrying' % f['type'])
time.sleep(1)
# confirm single file upload to bkit server
upload_done_url = paths.get_bkit_url() + 'uploads_s3/' + upload['id'] + '/upload-file/'
upload_response = requests.post(upload_done_url, headers=headers, verify=True)
bg_blender.progress('finished uploading')
return {'FINISHED'}
def verification_status_change(self, context, asset_id, state):
user_preferences = bpy.context.preferences.addons['blenderkit'].preferences
upload_data = {"verificationStatus": state}
url = paths.get_api_url() + 'assets/' + str(asset_id) + '/'
headers = utils.get_headers(user_preferences.api_key)
try:
r = requests.patch(url, json=upload_data, headers=headers,
verify=True) # files = files,
#print('changed status ')
#print(r.text)
except requests.exceptions.RequestException as e:
print(e)
return {'CANCELLED'}
return {'FINISHED'}
def update_ratings_work_hours(self, context):
user_preferences = bpy.context.preferences.addons['blenderkit'].preferences
api_key = user_preferences.api_key
headers = utils.get_headers(api_key)
asset = self.id_data
bkit_ratings = asset.bkit_ratings
url = paths.get_api_url(
) + 'assets/' + asset['asset_data']['id'] + '/rating/'
if bkit_ratings.rating_quality > 0.1:
ratings = [('working_hours', round(bkit_ratings.rating_work_hours, 1))]
tasks_queue.add_task(
(send_rating_to_thread_work_hours, (url, ratings, headers)),
wait=1,
only_last=True)
def upload_rating(asset):
user_preferences = bpy.context.preferences.addons['blenderkit'].preferences
api_key = user_preferences.api_key
headers = utils.get_headers(api_key)
bkit_ratings = asset.bkit_ratings
# print('rating asset', asset_data['name'], asset_data['assetBaseId'])
url = paths.get_api_url(
) + 'assets/' + asset['asset_data']['id'] + '/rating/'
ratings = []
if bkit_ratings.rating_quality > 0.1:
ratings = (('quality', bkit_ratings.rating_quality), )
tasks_queue.add_task(
(send_rating_to_thread_quality, (url, ratings, headers)),
wait=2.5,
only_last=True)
if bkit_ratings.rating_work_hours > 0.1:
ratings = (('working_hours', round(bkit_ratings.rating_work_hours,
1)), )
tasks_queue.add_task(
(send_rating_to_thread_work_hours, (url, ratings, headers)),
wait=2.5,
only_last=True)
thread = threading.Thread(target=upload_rating_thread,
args=(url, ratings, headers))
thread.start()
url = paths.get_api_url(
) + 'assets/' + asset['asset_data']['id'] + '/review'
reviews = {
'reviewText': bkit_ratings.rating_compliments,
'reviewTextProblems': bkit_ratings.rating_problems,
}
if not (bkit_ratings.rating_compliments == ''
and bkit_ratings.rating_compliments == ''):
thread = threading.Thread(target=upload_review_thread,
args=(url, reviews, headers))
thread.start()
# the info that the user rated an item is stored in the scene
s = bpy.context.scene
s['assets rated'] = s.get('assets rated', {})
if bkit_ratings.rating_quality > 0.1 and bkit_ratings.rating_work_hours > 0.1:
s['assets rated'][asset['asset_data']['assetBaseId']] = True
def update_ratings_work_hours(self, context):
user_preferences = bpy.context.preferences.addons['blenderkit'].preferences
api_key = user_preferences.api_key
headers = utils.get_headers(api_key)
asset = self.id_data
if asset:
bkit_ratings = asset.bkit_ratings
url = paths.get_api_url() + 'assets/' + asset['asset_data']['id'] + '/rating/'
else:
# this part is for operator rating:
bkit_ratings = self
url = paths.get_api_url() + f'assets/{self.asset_id}/rating/'
if bkit_ratings.rating_work_hours > 0.45:
ratings = [('working_hours', round(bkit_ratings.rating_work_hours, 1))]
tasks_queue.add_task((send_rating_to_thread_work_hours, (url, ratings, headers)), wait=2.5, only_last=True)
def patch_asset_empty(asset_id, api_key):
'''
This function patches the asset for the purpose of it getting a reindex.
Should be removed once this is fixed on the server and
the server is able to reindex after uploads of resolutions
Returns
-------
'''
upload_data = {
}
url = paths.get_api_url() + 'assets/' + str(asset_id) + '/'
headers = utils.get_headers(api_key)
try:
r = rerequests.patch(url, json=upload_data, headers=headers, verify=True) # files = files,
except requests.exceptions.RequestException as e:
print(e)
return {'CANCELLED'}
return {'FINISHED'}
def get_download_url(asset_data, scene_id, api_key, tcom=None):
''''retrieves the download url. The server checks if user can download the item.'''
mt = time.time()
headers = utils.get_headers(api_key)
data = {'scene_uuid': scene_id}
r = None
try:
r = requests.get(asset_data['download_url'],
params=data,
headers=headers)
except Exception as e:
print(e)
if tcom is not None:
tcom.error = True
if r == None:
tcom.report = 'Connection Error'
tcom.error = True
return 'Connection Error'
if r.status_code < 400:
data = r.json()
url = data['filePath']
asset_data['url'] = url
asset_data['file_name'] = paths.extract_filename_from_url(url)
return True
if r.status_code == 403:
r = 'You need Standard plan to get this item.'
tcom.report = r
r1 = 'All materials and brushes are aviable for free. Only users registered to Standart plan can use all models.'
tasks_queue.add_task((ui.add_report, (r1, 5, colors.RED)))
tcom.error = True
if r.status_code == 401:
tcom.report = 'Invalid API key'
tcom.error = True
return 'Invalid API key'
elif r.status_code >= 500:
tcom.report = 'Server error'
tcom.error = True
return False
def execute(self, context):
user_preferences = bpy.context.preferences.addons[
'blenderkit'].preferences
api_key = user_preferences.api_key
headers = utils.get_headers(api_key)
url = paths.get_api_url() + f'assets/{self.asset_id}/rating/'
rtgs = []
self.rating_quality = int(self.rating_quality_ui)
if self.rating_quality > 0.1:
rtgs.append(('quality', self.rating_quality))
if self.rating_work_hours > 0.1:
rtgs.append(('working_hours', round(self.rating_work_hours, 1)))
thread = threading.Thread(target=upload_rating_thread,
args=(url, rtgs, headers))
thread.start()
return {'FINISHED'}
def get_download_url(asset_data, scene_id, api_key, tcom=None):
''''retrieves the download url. The server checks if user can download the item.'''
mt = time.time()
headers = utils.get_headers(api_key)
data = {'scene_uuid': scene_id}
r = None
try:
r = requests.get(asset_data['download_url'],
params=data,
headers=headers)
except Exception as e:
print(e)
if tcom is not None:
tcom.error = True
if r == None:
tcom.report = 'Connection Error'
tcom.error = True
return 'Connection Error'
if r.status_code < 400:
data = r.json()
url = data['filePath']
asset_data['url'] = url
asset_data['file_name'] = paths.extract_filename_from_url(url)
return True
if r.status_code == 403:
tcom.report = 'Available only in higher plans.'
tcom.error = True
return 'Available only in higher plans.'
if r.status_code == 401:
tcom.report = 'Invalid API key'
tcom.error = True
return 'Invalid API key'
elif r.status_code >= 500:
tcom.report = 'Server error'
tcom.error = True
def mark_for_validation(self, context, asset_type):
props = utils.get_upload_props()
props.upload_state = 'marking for validation'
user_preferences = bpy.context.preferences.addons['blenderkit'].preferences
upload_data = {"verificationStatus": "ready"}
url = paths.get_api_url() + 'assets/'
headers = utils.get_headers(user_preferences.api_key)
url += props.id + '/'
try:
r = requests.patch(url, json=upload_data, headers=headers,
verify=True) # files = files,
props.upload_state = 'marked for validation'
except requests.exceptions.RequestException as e:
props.upload_state = str(e)
props.uploading = False
return {'CANCELLED'}
return {'FINISHED'}
def fetch_categories(API_key):
url = paths.get_api_url() + 'categories/'
headers = utils.get_headers(API_key)
tempdir = paths.get_temp_dir()
categories_filepath = os.path.join(tempdir, 'categories.json')
try:
r = rerequests.get(url, headers=headers)
rdata = r.json()
categories = rdata['results']
fix_category_counts(categories)
# filter_categories(categories) #TODO this should filter categories for search, but not for upload. by now off.
with open(categories_filepath, 'w') as s:
json.dump(categories, s, indent=4)
tasks_queue.add_task((load_categories, ()))
except Exception as e:
utils.p('category fetching failed')
utils.p(e)
if not os.path.exists(categories_filepath):
source_path = paths.get_addon_file(subpath='data' + os.sep + 'categories.json')
shutil.copy(source_path, categories_filepath)
def fetch_categories(API_key):
BLENDERKIT_API_MAIN = "https://www.blenderkit.com/api/v1/"
url = paths.get_bkit_url() + 'categories/'
headers = utils.get_headers(API_key)
tempdir = paths.get_temp_dir()
categories_filepath = os.path.join(tempdir, 'categories.json')
try:
r = requests.get(url, headers=headers)
rdata = r.json()
categories = rdata['results']
fix_category_counts(categories)
# filter_categories(categories) #TODO this should filter categories for search, but not for upload. by now off.
with open(categories_filepath, 'w') as s:
json.dump(categories, s, indent=4)
except:
# print('category fetching failed')
if not os.path.exists(categories_filepath):
source_path = paths.get_addon_file(subpath='data' + os.sep +
'categories.json')
shutil.copy(source_path, categories_filepath)
def report_usages():
'''report the usage of assets to the server.'''
mt = time.time()
user_preferences = bpy.context.preferences.addons['blenderkit'].preferences
api_key = user_preferences.api_key
sid = get_scene_id()
headers = utils.get_headers(api_key)
url = paths.get_api_url() + paths.BLENDERKIT_REPORT_URL
assets = {}
asset_obs = []
scene = bpy.context.scene
asset_usages = {}
for ob in scene.collection.objects:
if ob.get('asset_data') != None:
asset_obs.append(ob)
for ob in asset_obs:
asset_data = ob['asset_data']
abid = asset_data['asset_base_id']
if assets.get(abid) is None:
asset_usages[abid] = {'count': 1}
assets[abid] = asset_data
else:
asset_usages[abid]['count'] += 1
# brushes
for b in bpy.data.brushes:
if b.get('asset_data') != None:
abid = b['asset_data']['asset_base_id']
asset_usages[abid] = {'count': 1}
assets[abid] = b['asset_data']
# materials
for ob in scene.collection.objects:
for ms in ob.material_slots:
m = ms.material
if m is not None and m.get('asset_data') is not None:
abid = m['asset_data']['asset_base_id']
if assets.get(abid) is None:
asset_usages[abid] = {'count': 1}
assets[abid] = m['asset_data']
else:
asset_usages[abid]['count'] += 1
assets_list = []
assets_reported = scene.get('assets reported', {})
new_assets_count = 0
for k in asset_usages.keys():
if k not in assets_reported.keys():
data = asset_usages[k]
list_item = {
'asset': k,
'usageCount': data['count'],
'proximitySet': data.get('proximity', [])
}
assets_list.append(list_item)
new_assets_count += 1
if k not in assets_reported.keys():
assets_reported[k] = True
scene['assets reported'] = assets_reported
if new_assets_count == 0:
utils.p('no new assets were added')
return
usage_report = {
'scene': sid,
'reportType': 'save',
'assetusageSet': assets_list
}
au = scene.get('assets used', {})
ad = scene.get('assets deleted', {})
ak = assets.keys()
for k in au.keys():
if k not in ak:
ad[k] = au[k]
else:
if k in ad:
ad.pop(k)
# scene['assets used'] = {}
for k in ak: # rewrite assets used.
scene['assets used'][k] = assets[k]
###########check ratings herer too:
scene['assets rated'] = scene.get('assets rated', {})
for k in assets.keys():
scene['assets rated'][k] = scene['assets rated'].get(k, False)
thread = threading.Thread(target=utils.requests_post_thread,
args=(url, usage_report, headers))
thread.start()
mt = time.time() - mt
print('report generation: ', mt)
def start_upload(self, context, asset_type, as_new, metadata_only):
'''start upload process, by processing data'''
props = utils.get_upload_props()
storage_quota_ok = check_storage_quota(props)
if not storage_quota_ok:
self.report({'ERROR_INVALID_INPUT'}, props.report)
return {'CANCELLED'}
location = get_upload_location(props)
props.upload_state = 'preparing upload'
auto_fix(asset_type = asset_type)
# do this for fixing long tags in some upload cases
props.tags = props.tags[:]
props.name = props.name.strip()
# TODO move this to separate function
# check for missing metadata
if asset_type == 'MODEL':
get_missing_data_model(props)
if asset_type == 'SCENE':
get_missing_data_scene(props)
elif asset_type == 'MATERIAL':
get_missing_data_material(props)
elif asset_type == 'BRUSH':
get_missing_data_brush(props)
if props.report != '':
self.report({'ERROR_INVALID_INPUT'}, props.report)
return {'CANCELLED'}
if as_new:
props.asset_base_id = ''
props.id = ''
export_data, upload_data, eval_path_computing, eval_path_state, eval_path, props = get_upload_data(self, context,
asset_type)
# utils.pprint(upload_data)
upload_data['parameters'] = params_to_dict(
upload_data['parameters']) # weird array conversion only for upload, not for tooltips.
binary_path = bpy.app.binary_path
script_path = os.path.dirname(os.path.realpath(__file__))
basename, ext = os.path.splitext(bpy.data.filepath)
# if not basename:
# basename = os.path.join(basename, "temp")
if not ext:
ext = ".blend"
tempdir = tempfile.mkdtemp()
source_filepath = os.path.join(tempdir, "export_blenderkit" + ext)
clean_file_path = paths.get_clean_filepath()
data = {
'clean_file_path': clean_file_path,
'source_filepath': source_filepath,
'temp_dir': tempdir,
'export_data': export_data,
'upload_data': upload_data,
'debug_value': bpy.app.debug_value,
}
datafile = os.path.join(tempdir, BLENDERKIT_EXPORT_DATA_FILE)
# check if thumbnail exists:
if not os.path.exists(export_data["thumbnail_path"]):
props.upload_state = 'Thumbnail not found'
props.uploading = False
return {'CANCELLED'}
# first upload metadata to server, so it can be saved inside the current file
url = paths.get_api_url() + 'assets/'
headers = utils.get_headers(upload_data['token'])
# upload_data['license'] = 'ovejajojo'
json_metadata = upload_data # json.dumps(upload_data, ensure_ascii=False).encode('utf8')
global reports
if props.asset_base_id == '':
try:
r = requests.post(url, json=json_metadata, headers=headers, verify=True) # files = files,
props.upload_state = 'uploaded metadata'
utils.p(r.text)
except requests.exceptions.RequestException as e:
print(e)
props.upload_state = str(e)
props.uploading = False
return {'CANCELLED'}
else:
url += props.id + '/'
try:
if not metadata_only:
json_metadata["verificationStatus"] = "uploading"
r = requests.put(url, json=json_metadata, headers=headers, verify=True) # files = files,
props.upload_state = 'uploaded metadata'
# parse the reqest
# print('uploaded metadata')
# print(r.text)
except requests.exceptions.RequestException as e:
print(e)
props.upload_state = str(e)
props.uploading = False
return {'CANCELLED'}
# props.upload_state = 'step 1'
if metadata_only:
props.uploading = False
return {'FINISHED'}
try:
rj = r.json()
if props.asset_base_id == '':
props.asset_base_id = rj['assetBaseId']
props.id = rj['id']
upload_data['assetBaseId'] = props.asset_base_id
upload_data['id'] = props.id
bpy.ops.wm.save_mainfile()
# fa
props.uploading = True
# save a copy of actual scene but don't interfere with the users models
bpy.ops.wm.save_as_mainfile(filepath=source_filepath, compress=False, copy=True)
with open(datafile, 'w') as s:
json.dump(data, s)
proc = subprocess.Popen([
binary_path,
"--background",
"-noaudio",
clean_file_path,
"--python", os.path.join(script_path, "upload_bg.py"),
"--", datafile # ,filepath, tempdir
], bufsize=5000, stdout=subprocess.PIPE, stdin=subprocess.PIPE)
bg_blender.add_bg_process(eval_path_computing=eval_path_computing, eval_path_state=eval_path_state,
eval_path=eval_path, process_type='UPLOAD', process=proc, location=location)
except Exception as e:
props.upload_state = str(e)
props.uploading = False
return {'CANCELLED'}
return {'FINISHED'}
def run(self):
maxthreads = 300
maximages = 50
query = self.query
params = self.params
global reports
t = time.time()
mt('search thread started')
tempdir = paths.get_temp_dir('%s_search' % query['asset_type'])
json_filepath = os.path.join(
tempdir, '%s_searchresult.json' % query['asset_type'])
headers = utils.get_headers(params['api_key'])
rdata = {}
rdata['results'] = []
if params['get_next']:
with open(json_filepath, 'r') as infile:
try:
origdata = json.load(infile)
urlquery = origdata['next']
if urlquery == None:
return
except:
# in case no search results found on drive we don't do next page loading.
params['get_next'] = False
if not params['get_next']:
# build a new request
url = paths.get_api_url() + 'search/'
# build request manually
# TODO use real queries
requeststring = '?query=' + query['keywords'].lower() + '+'
#
for i, q in enumerate(query):
requeststring += q + ':' + str(query[q]).lower()
if i < len(query) - 1:
requeststring += '+'
requeststring += '&addon_version=%s' % params['addon_version']
if params.get('scene_uuid') is not None:
requeststring += '&scene_uuid=%s' % params['scene_uuid']
urlquery = url + requeststring
try:
utils.p(urlquery)
r = requests.get(urlquery, headers=headers)
reports = ''
# utils.p(r.text)
except requests.exceptions.RequestException as e:
print(e)
reports = e
# props.report = e
return
mt('response is back ')
try:
rdata = r.json()
except Exception as inst:
reports = r.text
print(inst)
mt('data parsed ')
# filter results here:
# todo remove this in future
nresults = []
for d in rdata.get('results', []):
# TODO this code is for filtering brush types, should vanish after we implement filter in Elastic
mode = None
if query['asset_type'] == 'brush':
for p in d['parameters']:
if p['parameterType'] == 'mode':
mode = p['value']
if query['asset_type'] != 'brush' or (
query.get('brushType') != None
and query['brushType']) == mode:
nresults.append(d)
rdata['results'] = nresults
# print('number of results: ', len(rdata.get('results', [])))
if self.stopped():
utils.p('stopping search : ' + query['keywords'])
return
mt('search finished')
i = 0
thumb_small_urls = []
thumb_small_filepaths = []
thumb_full_urls = []
thumb_full_filepaths = []
# END OF PARSING
getting_authors = {}
for d in rdata.get('results', []):
if getting_authors.get(d['author']['id']) is None:
get_author(d)
getting_authors[d['author']['id']] = True
for f in d['files']:
# TODO move validation of published assets to server, too manmy checks here.
if f['fileType'] == 'thumbnail' and f[
'fileThumbnail'] != None and f[
'fileThumbnailLarge'] != None:
if f['fileThumbnail'] == None:
f['fileThumbnail'] = 'NONE'
if f['fileThumbnailLarge'] == None:
f['fileThumbnailLarge'] = 'NONE'
thumb_small_urls.append(f['fileThumbnail'])
thumb_full_urls.append(f['fileThumbnailLarge'])
imgname = paths.extract_filename_from_url(
f['fileThumbnail'])
imgpath = os.path.join(tempdir, imgname)
thumb_small_filepaths.append(imgpath)
imgname = paths.extract_filename_from_url(
f['fileThumbnailLarge'])
imgpath = os.path.join(tempdir, imgname)
thumb_full_filepaths.append(imgpath)
sml_thbs = zip(thumb_small_filepaths, thumb_small_urls)
full_thbs = zip(thumb_full_filepaths, thumb_full_urls)
# we save here because a missing thumbnail check is in the previous loop
# we can also prepend previous results. These have allready thumbnails downloaded...
if params['get_next']:
rdata['results'][0:0] = origdata['results']
with open(json_filepath, 'w') as outfile:
json.dump(rdata, outfile)
killthreads_sml = []
for k in thumb_sml_download_threads.keys():
if k not in thumb_small_filepaths:
killthreads_sml.append(k) # do actual killing here?
killthreads_full = []
for k in thumb_full_download_threads.keys():
if k not in thumb_full_filepaths:
killthreads_full.append(k) # do actual killing here?
# TODO do the killing/ stopping here! remember threads might have finished inbetween!
if self.stopped():
utils.p('stopping search : ' + query['keywords'])
return
# this loop handles downloading of small thumbnails
for imgpath, url in sml_thbs:
if imgpath not in thumb_sml_download_threads and not os.path.exists(
imgpath):
thread = ThumbDownloader(url, imgpath)
# thread = threading.Thread(target=download_thumbnail, args=([url, imgpath]),
# daemon=True)
thread.start()
thumb_sml_download_threads[imgpath] = thread
# threads.append(thread)
if len(thumb_sml_download_threads) > maxthreads:
while len(thumb_sml_download_threads) > maxthreads:
threads_copy = thumb_sml_download_threads.copy(
) # because for loop can erase some of the items.
for tk, thread in threads_copy.items():
if not thread.is_alive():
thread.join()
# utils.p(x)
del (thumb_sml_download_threads[tk])
# utils.p('fetched thumbnail ', i)
i += 1
if self.stopped():
utils.p('stopping search : ' + query['keywords'])
return
idx = 0
while len(thumb_sml_download_threads) > 0:
threads_copy = thumb_sml_download_threads.copy(
) # because for loop can erase some of the items.
for tk, thread in threads_copy.items():
if not thread.is_alive():
thread.join()
del (thumb_sml_download_threads[tk])
i += 1
if self.stopped():
utils.p('stopping search : ' + query['keywords'])
return
# start downloading full thumbs in the end
for imgpath, url in full_thbs:
if imgpath not in thumb_full_download_threads and not os.path.exists(
imgpath):
thread = ThumbDownloader(url, imgpath)
# thread = threading.Thread(target=download_thumbnail, args=([url, imgpath]),
# daemon=True)
thread.start()
thumb_full_download_threads[imgpath] = thread
mt('thumbnails finished')
