def handle_api_result(self, info, kwargs=None, token=None): if token is None: token = self.wait_token() try: userinfo = info['query']['userinfo'] except KeyError: userinfo = () if 'blockedby' in userinfo: self.blocked = (userinfo['blockedby'], userinfo.get('blockreason', u'')) else: self.blocked = False self.hasmsg = 'message' in userinfo self.logged_in = 'anon' not in userinfo if 'error' in info: if info['error']['code'] in ( u'internal_api_error_DBConnectionError', u'internal_api_error_DBQueryError'): self.wait(token) return False if '*' in info['error']: raise errors.APIError(info['error']['code'], info['error']['info'], info['error']['*']) raise errors.APIError(info['error']['code'], info['error']['info'], kwargs) return True
def get_render(result, media_id, owner_username): if result[0] != 0: raise errors.APIError(result[1]) if len(result[1]) > 0: d3 = self._make_media_path(media_id, result[1][0], owner_username, width, height, crop) d3.addCallback(get_binary_data) return d3 else: raise errors.APIError("Render supposedly exists, but I couldn't find any nodes.")
def handle_nodes(result, media_id, owner_username): """ I don't know what the hell this does. looks like nothing. @return: Unknown @rtype: Unknown The above comment was added by Clint. I left it here to illustrate something: Clint's full of shit. V """ if result[0] != 0: raise errors.APIError(result[1]) nodes = result[1] dl = [] for n in nodes: d2 = self._make_media_path(media_id, n, owner_username) d2.addCallback(store) d2.addCallback(lambda _: self.clear_renders(media_id, owner_username, n)) dl.append(d2) dList = DeferredList(dl) dList.addCallback(lambda _: "success") return dList
def getFacility(facilityId): response = FacilityTable.get_item(Key={"facilityId": facilityId}) try: facilityData = response["Item"] except KeyError: raise errors.APIError("No facility found for id: " + str(facilityId)) return loadFacility(facilityData)
def store(result): if result[0] != 0: raise errors.APIError(result[1]) path = result[1] self.log.debug("writing to path: %s" % path) return self._write_binary("%s.jpg" % path, data, True)
def get_binary_data(result): if result[0] != 0: raise errors.APIError(result[1]) d4 = self._read_binary("%s.jpg" % result[1]) d4.addCallback(lambda _: _[1]) return d4
def handle_servers(servers): if len(servers) < distribution_total: raise errors.APIError("All servers are full or busted!") hosts = [] for s in servers: hosts.append(s['hostname']) self.log.debug("servers chosen for storage were: %s" % pformat(hosts)) return hosts
def do_read(file): if os.path.exists(file): f = open(file, "rb") data = f.read() f.close() return data else: raise errors.APIError("File not found: %s" % file)
def handle_servers(result, media_id, owner_username): if result[0] != 0: raise errors.APIError(result[1]) node = result[1][0] d2 = self._make_media_path(media_id, node, owner_username) d2.addCallback(check_exists) return d2
def check_exists(result): if result[0] == 0: if not result[1]: return self._find_suitable_storage_targets(id, data, self._cfg_default_distribution_total) else: return 0 else: raise errors.APIError(result[1])
def handle_servers(result): if result[0] != 0: raise errors.APIError(result[1]) host = result[1][0] d2 = self._make_media_path(media_id, host, username) d2.addCallback(check_user_path, host) return d2
def do_delete(void): if os.path.exists(filename): self.log.debug("file [%s] exists...deleting" % filename) try: os.unlink(filename) except Exception, ex: self.log.warning("Unable to delete [%s] - %s" % (filename, ex)) raise errors.APIError(ex)
def check_file_exists(void): if os.path.exists(filename): if delete: return self._delete_binary(filename) else: raise errors.APIError("File %s already exists, and delete is False!" % filename) else: self.log.debug("file %s doesn't exist" % filename) return (0, filename)
def check_renders(result, media_id, owner_username): if result[0] != 0: raise errors.APIError(result[1]) d2 = Deferred() for node in result[1]: d2.addCallback(get_node_path, node, media_id, owner_username) d2.callback(False) return d2
def extract_data(result): if result[0] != 0: raise errors.APIError(result[1]) media_path = result[1] self.log.debug("get_media_info() reading media from %s" % media_path) d4 = self._read_binary("%s.jpg" % media_path) d4.addCallback(build_return, media_path) return d4
def check_user_path(result, host): if result[0] != 0: raise errors.APIError(result[1]) if os.path.exists("%s.jpg" % result[1]): return extract_data(result) else: d3 = self._make_media_path(media_id, host) d3.addCallback(extract_data) return d3
def check_render_exists(result): if result[0] != 0: raise errors.APIError(result[1]) if result[1]: d2 = self.app.api.images.get_media_owner_id(image_id) d2.addCallback(handle_media_result) return d2 else: return (0, None)
def api(self, action, *args, **kwargs): kwargs.update(args) if action == 'query': if 'meta' in kwargs: kwargs['meta'] += '|userinfo' else: kwargs['meta'] = 'userinfo' if 'uiprop' in kwargs: kwargs['uiprop'] += '|blockinfo|hasmsg' else: kwargs['uiprop'] = 'blockinfo|hasmsg' token = self.wait_token() while True: info = self.raw_api(action, **kwargs) if not info: info = {} try: userinfo = compatibility.userinfo( info, self.require(1, 12, raise_error=None)) except KeyError: userinfo = () if 'blockedby' in userinfo: self.blocked = (userinfo['blockedby'], userinfo.get('blockreason', u'')) else: self.blocked = False self.hasmsg = 'message' in userinfo self.logged_in = 'anon' not in userinfo if 'error' in info: if info['error']['code'] in ( u'internal_api_error_DBConnectionError', ): self.wait(token) continue if '*' in info['error']: raise errors.APIError(info['error']['code'], info['error']['info'], info['error']['*']) raise errors.APIError(info['error']['code'], info['error']['info'], kwargs) return info
def do_clear(result): if result[0] != 0: raise errors.APIError(result[1]) paths = result[1] dl = [] for path in paths: self.log.debug("running delete on [%s.jpg]" % path) dl.append(self._delete_binary("%s.jpg" % path)) dList = DeferredList(dl) dList.addCallback(lambda _: "success") return dList
def delete_image(result, media_id, owner_username): if result[0] != 0: raise errors.APIError(result[1]) nodes = result[1] dl2 = [] for n in nodes: d2 = self._make_media_path(media_id, n, owner_username) d2.addCallback(delete_on_node) d2.addCallback(wipe_renders, media_id, owner_username, n) dl2.append(d2) dList2 = DeferredList(dl2) dList2.addCallback(lambda _: "success") return dList2
def handle_nodes(result, media_id, owner_username): if result[0] != 0: raise errors.APIError(result[1]) nodes = result[1] self.log.debug("got nodes %s from locate_media()" % pformat(nodes)) dl = [] for n in nodes: self.log.debug("storing media %s on node %s" % (media_id, n)) d2 = self._make_media_path(media_id, n, owner_username, width, height, crop) d2.addCallback(store) dl.append(d2) dList = DeferredList(dl) dList.addCallback(lambda _: data) return dList
def build_return(result, path): if result[0] != 0: raise errors.APIError(result[1]) data = result[1] file_mtime = datetime.fromtimestamp(os.stat("%s.jpg" % path).st_mtime) rval = { 'media_id': media_id, 'media_type_id': 1, 'size_B': len(data), 'updated': file_mtime, 'stored_by': username or 'aztk' } if binary_data: rval['data'] = data return rval
def main_inserts(result): if result[0] != 0: self.log.warning("Error getting storage_targets: %s" % result[1]) raise errors.APIError(result[1]) nodes = result[1] self.log.debug("Got %s nodes back from _find_suitable()" % pformat(nodes)) dl = [] for n in nodes: self.log.debug("getting path for image %s, node %s" % (id, n)) d2 = self._make_media_path(id, n) d2.addCallback(handle_path) d2.addCallback(lambda _: dl.append(self.app.db.runInteraction(insert_txn, n, id, host=n))) dl.append(d2) d_list = DeferredList(dl, fireOnOneErrback=1) d_list.addCallback(lambda _: 0) return d_list
def check_insert(result): if not result: raise errors.APIError( "Trying to process user %s, but no record exists!" % user_info['username']) query = """ UPDATE users SET email = %(email)s, first_name = %(first_name)s, last_name = %(last_name)s, gender = %(gender)s, email_upload_key = %(email_key)s, last_login = %(last_login)s, date_created = %(date_created)s, bio = %(bio)s, flags = %(flags)s, successful_invites = %(successful_invites)s, extra_storage_MB = %(extra_storage)s, country = %(country)s, zip = %(zip)s WHERE username = %(username)s """ self.log.debug(query) def update_txn(txn, info): txn.execute(query, info) txn.execute( "SELECT userid FROM users WHERE username = %(username)s", info) info['userid'] = txn.fetchone()[0] txn.execute("SELECT zoto_init_user(%(userid)s)", info) def handle_fail(fail, query): raise errors.APIError("""Error updating user: %s query= %s """ % (fail.getErrorMessage(), query)) d2 = self.app.db.runInteraction(update_txn, user_info) d2.addErrback(handle_fail, query) return d2
def create_or_reuse_task(sequence, parameters, reusable_tasks, task_key_params, validate_task_output): new_task_attrs = { 'job_uuid': arvados.current_job()['uuid'], 'created_by_job_task_uuid': arvados.current_task()['uuid'], 'sequence': sequence, 'parameters': parameters } # See if there is a task in reusable_tasks that can be reused ct_index = tuple( [parameters[index_param] for index_param in task_key_params]) if len(reusable_tasks) == 0: print "No reusable tasks were available" elif ct_index in reusable_tasks: # have a task from which to reuse the output, prepare to create a new, but already finished, task with that output reuse_task = reusable_tasks[ct_index] if validate_task_output(reuse_task['output']): print "Found existing JobTask %s from Job %s. Will use output %s from that JobTask instead of re-running it." % ( reuse_task['uuid'], reuse_task['job_uuid'], reuse_task['output']) # remove task from reusable_tasks as it won't be used more than once del reusable_tasks[ct_index] # copy relevant attrs from reuse_task so that the new tasks start already finished for attr in [ 'success', 'output', 'progress', 'started_at', 'finished_at', 'parameters' ]: new_task_attrs[attr] = reuse_task[attr] # crunch seems to ignore the fact that the job says it is done and queue it anyway # signal ourselves to just immediately exit successfully when we are run new_task_attrs['parameters']['reuse_job_task'] = reuse_task['uuid'] else: print "Output %s for potential task reuse did not validate" % ( reuse_task['output']) else: print "No reusable JobTask matched key parameters %s" % ( list(ct_index)) # Create the "new" task (may be new work or may be already finished work) new_task = arvados.api().job_tasks().create(body=new_task_attrs).execute() if not new_task: raise errors.APIError("Attempt to create new job_task failed: [%s]" % new_task_attrs) return new_task
def check_path_exists(result): if result[0] != 0: self.log.debug("error in check_path_exists: %s" % result[1]) raise errors.APIError(result[1]) dir_name = os.path.dirname(filename) self.log.debug("dir_name: %s" % dir_name) if not os.path.exists(dir_name): self.log.debug("path doesn't exist...creating") try: os.makedirs(dir_name) except Exception, ex: if ex.errno == 17: ## must have been created by another process...ignore! pass else: self.log.warning("Error making output directory: %s" % ex) raise ex self.log.debug("path created")
def add_album(result, album, images): if result[0] != 0: raise errors.APIError(result[1]) if result[1]: album_id = result[1] self.log.debug( "album already exists with id [%s]...adding images" % album_id) d2 = Deferred() d2.callback((0, album_id)) else: self.log.debug("Creating album: %s" % album['title']) album_info = { 'title': album['title'], 'description': album['description'], 'main_image': album['main_image'], 'order_by': album['order_by'], 'order_dir': album['order_dir'] } d2 = self.app.api.albums.create_album(user_info['userid'], album_info) d2.addCallback(add_images, album, images) return d2
def handle_fail(fail, query): raise errors.APIError("""Error updating user: %s query= %s """ % (fail.getErrorMessage(), query))
def get_count(result): if result: return result['count'] else: raise errors.APIError("Error getting count")
def wipe_renders(result, media_id, owner_username, node): if result[0] != 0: raise errors.APIError(result[1]) return self.clear_renders(media_id, owner_username, node)