def handle_api_result(self, info, kwargs=None, token=None):
if token is None:
token = self.wait_token()
try:
userinfo = info['query']['userinfo']
except KeyError:
userinfo = ()
if 'blockedby' in userinfo:
self.blocked = (userinfo['blockedby'],
userinfo.get('blockreason', u''))
else:
self.blocked = False
self.hasmsg = 'message' in userinfo
self.logged_in = 'anon' not in userinfo
if 'error' in info:
if info['error']['code'] in (
u'internal_api_error_DBConnectionError',
u'internal_api_error_DBQueryError'):
self.wait(token)
return False
if '*' in info['error']:
raise errors.APIError(info['error']['code'],
info['error']['info'],
info['error']['*'])
raise errors.APIError(info['error']['code'], info['error']['info'],
kwargs)
return True
def get_render(result, media_id, owner_username):
if result[0] != 0:
raise errors.APIError(result[1])
if len(result[1]) > 0:
d3 = self._make_media_path(media_id, result[1][0], owner_username, width, height, crop)
d3.addCallback(get_binary_data)
return d3
else:
raise errors.APIError("Render supposedly exists, but I couldn't find any nodes.")
def handle_nodes(result, media_id, owner_username): """ I don't know what the hell this does. looks like nothing. @return: Unknown @rtype: Unknown The above comment was added by Clint. I left it here to illustrate something: Clint's full of shit. V """ if result[0] != 0: raise errors.APIError(result[1]) nodes = result[1] dl = [] for n in nodes: d2 = self._make_media_path(media_id, n, owner_username) d2.addCallback(store) d2.addCallback(lambda _: self.clear_renders(media_id, owner_username, n)) dl.append(d2) dList = DeferredList(dl) dList.addCallback(lambda _: "success") return dList
def getFacility(facilityId):
response = FacilityTable.get_item(Key={"facilityId": facilityId})
try:
facilityData = response["Item"]
except KeyError:
raise errors.APIError("No facility found for id: " + str(facilityId))
return loadFacility(facilityData)
def store(result):
if result[0] != 0:
raise errors.APIError(result[1])
path = result[1]
self.log.debug("writing to path: %s" % path)
return self._write_binary("%s.jpg" % path, data, True)
def get_binary_data(result):
if result[0] != 0:
raise errors.APIError(result[1])
d4 = self._read_binary("%s.jpg" % result[1])
d4.addCallback(lambda _: _[1])
return d4
def handle_servers(servers):
if len(servers) < distribution_total:
raise errors.APIError("All servers are full or busted!")
hosts = []
for s in servers:
hosts.append(s['hostname'])
self.log.debug("servers chosen for storage were: %s" % pformat(hosts))
return hosts
def do_read(file):
if os.path.exists(file):
f = open(file, "rb")
data = f.read()
f.close()
return data
else:
raise errors.APIError("File not found: %s" % file)
def handle_servers(result, media_id, owner_username): if result[0] != 0: raise errors.APIError(result[1]) node = result[1][0] d2 = self._make_media_path(media_id, node, owner_username) d2.addCallback(check_exists) return d2
def check_exists(result): if result[0] == 0: if not result[1]: return self._find_suitable_storage_targets(id, data, self._cfg_default_distribution_total) else: return 0 else: raise errors.APIError(result[1])
def handle_servers(result): if result[0] != 0: raise errors.APIError(result[1]) host = result[1][0] d2 = self._make_media_path(media_id, host, username) d2.addCallback(check_user_path, host) return d2
def do_delete(void):
if os.path.exists(filename):
self.log.debug("file [%s] exists...deleting" % filename)
try:
os.unlink(filename)
except Exception, ex:
self.log.warning("Unable to delete [%s] - %s" % (filename, ex))
raise errors.APIError(ex)
def check_file_exists(void):
if os.path.exists(filename):
if delete:
return self._delete_binary(filename)
else:
raise errors.APIError("File %s already exists, and delete is False!" % filename)
else:
self.log.debug("file %s doesn't exist" % filename)
return (0, filename)
def check_renders(result, media_id, owner_username): if result[0] != 0: raise errors.APIError(result[1]) d2 = Deferred() for node in result[1]: d2.addCallback(get_node_path, node, media_id, owner_username) d2.callback(False) return d2
def extract_data(result):
if result[0] != 0:
raise errors.APIError(result[1])
media_path = result[1]
self.log.debug("get_media_info() reading media from %s" % media_path)
d4 = self._read_binary("%s.jpg" % media_path)
d4.addCallback(build_return, media_path)
return d4
def check_user_path(result, host):
if result[0] != 0:
raise errors.APIError(result[1])
if os.path.exists("%s.jpg" % result[1]):
return extract_data(result)
else:
d3 = self._make_media_path(media_id, host)
d3.addCallback(extract_data)
return d3
def check_render_exists(result): if result[0] != 0: raise errors.APIError(result[1]) if result[1]: d2 = self.app.api.images.get_media_owner_id(image_id) d2.addCallback(handle_media_result) return d2 else: return (0, None)
def api(self, action, *args, **kwargs):
kwargs.update(args)
if action == 'query':
if 'meta' in kwargs:
kwargs['meta'] += '|userinfo'
else:
kwargs['meta'] = 'userinfo'
if 'uiprop' in kwargs:
kwargs['uiprop'] += '|blockinfo|hasmsg'
else:
kwargs['uiprop'] = 'blockinfo|hasmsg'
token = self.wait_token()
while True:
info = self.raw_api(action, **kwargs)
if not info: info = {}
try:
userinfo = compatibility.userinfo(
info, self.require(1, 12, raise_error=None))
except KeyError:
userinfo = ()
if 'blockedby' in userinfo:
self.blocked = (userinfo['blockedby'],
userinfo.get('blockreason', u''))
else:
self.blocked = False
self.hasmsg = 'message' in userinfo
self.logged_in = 'anon' not in userinfo
if 'error' in info:
if info['error']['code'] in (
u'internal_api_error_DBConnectionError', ):
self.wait(token)
continue
if '*' in info['error']:
raise errors.APIError(info['error']['code'],
info['error']['info'],
info['error']['*'])
raise errors.APIError(info['error']['code'],
info['error']['info'], kwargs)
return info
def do_clear(result):
if result[0] != 0:
raise errors.APIError(result[1])
paths = result[1]
dl = []
for path in paths:
self.log.debug("running delete on [%s.jpg]" % path)
dl.append(self._delete_binary("%s.jpg" % path))
dList = DeferredList(dl)
dList.addCallback(lambda _: "success")
return dList
def delete_image(result, media_id, owner_username): if result[0] != 0: raise errors.APIError(result[1]) nodes = result[1] dl2 = [] for n in nodes: d2 = self._make_media_path(media_id, n, owner_username) d2.addCallback(delete_on_node) d2.addCallback(wipe_renders, media_id, owner_username, n) dl2.append(d2) dList2 = DeferredList(dl2) dList2.addCallback(lambda _: "success") return dList2
def handle_nodes(result, media_id, owner_username):
if result[0] != 0:
raise errors.APIError(result[1])
nodes = result[1]
self.log.debug("got nodes %s from locate_media()" % pformat(nodes))
dl = []
for n in nodes:
self.log.debug("storing media %s on node %s" % (media_id, n))
d2 = self._make_media_path(media_id, n, owner_username, width, height, crop)
d2.addCallback(store)
dl.append(d2)
dList = DeferredList(dl)
dList.addCallback(lambda _: data)
return dList
def build_return(result, path):
if result[0] != 0:
raise errors.APIError(result[1])
data = result[1]
file_mtime = datetime.fromtimestamp(os.stat("%s.jpg" % path).st_mtime)
rval = {
'media_id': media_id,
'media_type_id': 1,
'size_B': len(data),
'updated': file_mtime,
'stored_by': username or 'aztk'
}
if binary_data:
rval['data'] = data
return rval
def main_inserts(result):
if result[0] != 0:
self.log.warning("Error getting storage_targets: %s" % result[1])
raise errors.APIError(result[1])
nodes = result[1]
self.log.debug("Got %s nodes back from _find_suitable()" % pformat(nodes))
dl = []
for n in nodes:
self.log.debug("getting path for image %s, node %s" % (id, n))
d2 = self._make_media_path(id, n)
d2.addCallback(handle_path)
d2.addCallback(lambda _: dl.append(self.app.db.runInteraction(insert_txn, n, id, host=n)))
dl.append(d2)
d_list = DeferredList(dl, fireOnOneErrback=1)
d_list.addCallback(lambda _: 0)
return d_list
def check_insert(result):
if not result:
raise errors.APIError(
"Trying to process user %s, but no record exists!" %
user_info['username'])
query = """
UPDATE
users
SET
email = %(email)s,
first_name = %(first_name)s,
last_name = %(last_name)s,
gender = %(gender)s,
email_upload_key = %(email_key)s,
last_login = %(last_login)s,
date_created = %(date_created)s,
bio = %(bio)s,
flags = %(flags)s,
successful_invites = %(successful_invites)s,
extra_storage_MB = %(extra_storage)s,
country = %(country)s,
zip = %(zip)s
WHERE
username = %(username)s
"""
self.log.debug(query)
def update_txn(txn, info):
txn.execute(query, info)
txn.execute(
"SELECT userid FROM users WHERE username = %(username)s",
info)
info['userid'] = txn.fetchone()[0]
txn.execute("SELECT zoto_init_user(%(userid)s)", info)
def handle_fail(fail, query):
raise errors.APIError("""Error updating user: %s
query=
%s
""" % (fail.getErrorMessage(), query))
d2 = self.app.db.runInteraction(update_txn, user_info)
d2.addErrback(handle_fail, query)
return d2
def create_or_reuse_task(sequence, parameters, reusable_tasks, task_key_params,
validate_task_output):
new_task_attrs = {
'job_uuid': arvados.current_job()['uuid'],
'created_by_job_task_uuid': arvados.current_task()['uuid'],
'sequence': sequence,
'parameters': parameters
}
# See if there is a task in reusable_tasks that can be reused
ct_index = tuple(
[parameters[index_param] for index_param in task_key_params])
if len(reusable_tasks) == 0:
print "No reusable tasks were available"
elif ct_index in reusable_tasks:
# have a task from which to reuse the output, prepare to create a new, but already finished, task with that output
reuse_task = reusable_tasks[ct_index]
if validate_task_output(reuse_task['output']):
print "Found existing JobTask %s from Job %s. Will use output %s from that JobTask instead of re-running it." % (
reuse_task['uuid'], reuse_task['job_uuid'],
reuse_task['output'])
# remove task from reusable_tasks as it won't be used more than once
del reusable_tasks[ct_index]
# copy relevant attrs from reuse_task so that the new tasks start already finished
for attr in [
'success', 'output', 'progress', 'started_at',
'finished_at', 'parameters'
]:
new_task_attrs[attr] = reuse_task[attr]
# crunch seems to ignore the fact that the job says it is done and queue it anyway
# signal ourselves to just immediately exit successfully when we are run
new_task_attrs['parameters']['reuse_job_task'] = reuse_task['uuid']
else:
print "Output %s for potential task reuse did not validate" % (
reuse_task['output'])
else:
print "No reusable JobTask matched key parameters %s" % (
list(ct_index))
# Create the "new" task (may be new work or may be already finished work)
new_task = arvados.api().job_tasks().create(body=new_task_attrs).execute()
if not new_task:
raise errors.APIError("Attempt to create new job_task failed: [%s]" %
new_task_attrs)
return new_task
def check_path_exists(result):
if result[0] != 0:
self.log.debug("error in check_path_exists: %s" % result[1])
raise errors.APIError(result[1])
dir_name = os.path.dirname(filename)
self.log.debug("dir_name: %s" % dir_name)
if not os.path.exists(dir_name):
self.log.debug("path doesn't exist...creating")
try:
os.makedirs(dir_name)
except Exception, ex:
if ex.errno == 17:
## must have been created by another process...ignore!
pass
else:
self.log.warning("Error making output directory: %s" % ex)
raise ex
self.log.debug("path created")
def add_album(result, album, images):
if result[0] != 0:
raise errors.APIError(result[1])
if result[1]:
album_id = result[1]
self.log.debug(
"album already exists with id [%s]...adding images" %
album_id)
d2 = Deferred()
d2.callback((0, album_id))
else:
self.log.debug("Creating album: %s" % album['title'])
album_info = {
'title': album['title'],
'description': album['description'],
'main_image': album['main_image'],
'order_by': album['order_by'],
'order_dir': album['order_dir']
}
d2 = self.app.api.albums.create_album(user_info['userid'],
album_info)
d2.addCallback(add_images, album, images)
return d2
def handle_fail(fail, query):
raise errors.APIError("""Error updating user: %s
query=
%s
""" % (fail.getErrorMessage(), query))
def get_count(result):
if result:
return result['count']
else:
raise errors.APIError("Error getting count")
def wipe_renders(result, media_id, owner_username, node): if result[0] != 0: raise errors.APIError(result[1]) return self.clear_renders(media_id, owner_username, node)
