def get_service(): """ Returns core service data. At the moment this mainly means the central service CA cert. """ log = current_app.log ca_data = None client_conf = getConfig("client") ca_cert = client_conf.get("cafile", None) if ca_cert: try: with open(ca_cert, "r") as ca_fd: ca_data = ca_fd.read() except Exception as err: log.error("Failed to read cafile for service endpoint: %s", str(err)) # Get the user endpoint ep_conf = getConfig("endpoints") user_ep = ep_conf.get("users", None) # Build output dictionary res = {} if ca_data: res["central_ca"] = ca_data if user_ep: res["user_ep"] = user_ep if current_app.vo_list: res["vos"] = current_app.vo_list return jsonify(res)
def post_job(): """Add a job.""" require_attrs('src_siteid') Job = request.db.tables.Job # pylint: disable=invalid-name user_id = HRService.check_token() request.data['user_id'] = user_id request.data['src_credentials'] = current_app.site_client\ .get_cred(request.data['src_siteid'], user_id) if request.data['type'] == JobType.COPY: require_attrs('dst_siteid') request.data['dst_credentials'] = current_app.site_client\ .get_cred(request.data['dst_siteid'], user_id) elif request.data['type'] == JobType.RENAME and\ request.data.get('dst_siteid') != request.data['src_siteid']: current_app.log.warn("dst_siteid (%s) != src_siteid (%s)", request.data.get('dst_siteid'), request.data['src_siteid']) request.data['dst_siteid'] = request.data['src_siteid'] try: job = Job(**request.data) except ValueError as err: abort(400, description=err.message) except Exception as err: # pylint: disable=broad-except abort(500, description=err.message) try: job.add() except Exception as err: # pylint: disable=broad-except abort(500, description=err.message) return jsonify(job)
def get_site_list(): """ Get a list of all sites, although actually only returns entires that the user can see (public + self-owned). """ log = current_app.log db = request.db user_id = SiteService.get_current_uid() Site = db.tables.Site site_list = [] sites = Site.query.all() for site_entry in sites: # Ensure user can only see the correct sites is_owner = False is_visible = False if site_entry.site_owner == user_id: is_owner = True is_visible = True elif site_entry.public: is_visible = True if not is_visible: continue # Build the output object cur_site = {"is_owner": is_owner, "public": site_entry.public} for key in ("site_id", "site_name", "site_desc", "def_path"): cur_site[key] = getattr(site_entry, key) site_list.append(cur_site) log.info("Found %u sites for user id %u.", len(site_list), user_id) return jsonify(site_list)
def get_job_status(job_id): """Get job status.""" Job = request.db.tables.Job # pylint: disable=invalid-name job = Job.query.filter_by(id=job_id, user_id=HRService.check_token())\ .first_or_404() # pylint: disable=no-member return jsonify({'jobid': job.id, 'status': JobStatus(job.status).name})
def verify_token(): """ Return whether a client token is valid. """ if request.token_ok: res = "Token OK! (%s)" % request.token else: res = "Token Missing!" return jsonify(res)
def get_site(site_id): """ Gets the full details for a given site ID. Returns 404 if the site doesn't exist or if the user isn't allowed to view this site. """ log = current_app.log db = request.db user_id = SiteService.get_current_uid() Site = db.tables.Site site = Site.query.filter_by(site_id=site_id).first_or_404() is_owner = (site.site_owner == user_id) if not (site.public or is_owner): log.warn("User %u failed to get site %u details (no permission).", user_id, site_id) abort(404) # User isn't allowed to see this site log.info("User %u got details of site %u (%s).", user_id, site_id, site.site_name) # Add the endpoint info endpoints = [] for ep_info in site.endpoints: endpoints.append(ep_info.ep_uri) # Prepare the output data dict_out = dict(site) dict_out["is_owner"] = is_owner dict_out["endpoints"] = endpoints return jsonify(dict_out)
def verify_user(): """ Verify users' email address. Data posted is a token which has to ve verified and unpacked. The email address contained in the token is compared with the email stored in the DB. User state is changed to 1 (verified) if successful. :return: """ data = json.loads(request.data) HRService._logger.info("Data received for validation: %s", data) try: mtoken = data['mailtoken'] plain = current_app.mail_token_service.check(mtoken) HRService._logger.info("Mailer token verified OK: %s", plain) # token checked for integrity, check if not expired if HRUtils.is_date_passed(plain.get('expiry')): HRService._logger.error("Email verification token expired.") abort(400, "Bad token or already verified") username = plain.get('email') if not username: HRService._logger.error( "Email verification token does not contain user info.") abort(400, "Bad token or already verified") # 500? HRService.update_user_status(username, HRServiceUserState.VERIFIED) response = jsonify({'Verified': True, 'username': username}) response.status_code = 201 return response except ValueError as ve: HRService._logger.error( "Mailer token integrity verification failed (%s)", ve) abort(400, "Bad token or already verified")
def resend_email(): """ Re-send a verification email for registered but not verified users. :return: """ data = json.loads(request.data) if not 'email' in data: HRService._logger.error("resend email request:no email supplied") abort(400) username = data['email'] HRService._logger.info("Re-sending verification email for user %s .", username) User = request.db.tables.User user = User.query.filter_by(email=data['email']).first() if not user: # Raise an HTTPException with a 400 bad request status code HRService._logger.error( "resending email: requested user for id %s doesn't exist ", username) abort(400) if user.state != 0: HRService._logger.error( "resending email: requested user for id %s is already verified ", username) abort(400, ' Bad request or user already verified') HRService.email_user(user.email) HRService._logger.info("user: %s: verification email sent. ", user.email) response = jsonify([{'MailSent': 'OK'}]) response.status_code = 200 return response
def hello(): """ Ping-like method. :return: nice greeting ;-) """ return jsonify('User Service Desk at your service !\n')
def add_user(): """ Add a new user. :return: json document with added user data. An HTTPException with a 403 status code is thrown when the user already exists. User email address and a password are obligatory (404 is emitted otherwise). """ data = json.loads(request.data) if not 'email' in data: HRService._logger.error("add user request:no email supplied") abort(400) if not 'password' in data: HRService._logger.error("add user request:no password supplied") abort(400) if not HRService.check_passwd(data['password']): return "Password supplied is too short (lower character limit is %d)" \ % current_app.pwd_len, 400 data['password'] = hash_pass(data['password']) User = request.db.tables.User data.pop('last_login', None) data.pop('date_created', None) data.pop('date_modified', func.current_timestamp()) data.pop('state', 0) # user = User.from_json(json.dumps(data)) user = User(**data) db = request.db try: # user.save(db) db.session.add(user) user_id = db.session.query( User.id).filter_by(email=data['email']).scalar() # email the user HRService.email_user(user.email) HRService._logger.info("user: %s: verification email sent. ", user.email) db.session.commit() except RuntimeError as r_error: db.session.rollback() HRService._logger.error( "Runtime error when trying to send an email\n: %s", r_error) HRService._logger.error("User %s not added.", user.email) abort(500, 'The server could not send the verification email.') except Exception: HRService._logger.exception("Failed to add user: %s", user.name) db.session.rollback() abort(403) # 500 ? # dict response = jsonify(user) response.status_code = 201 return response
def get_element(job_id, element_id): """Get all jobs for a user.""" JobElement = request.db.tables.JobElement # pylint: disable=invalid-name element = JobElement.query.filter_by(id=element_id, job_id=job_id)\ .join(JobElement.job)\ .filter_by(user_id=HRService.check_token())\ .first_or_404() return jsonify(element)
def get_cred(site_id, user_id): """ Get a credential for a user at a specific site. """ log = current_app.log db = request.db Cred = db.tables.Cred cred = Cred.query.filter_by(cred_owner=user_id, site_id=site_id).first_or_404() log.info("Fetched cred for user %u at site %u.", user_id, site_id) return jsonify(cred.cred_value)
def test_output(self): # check the token is passed methodmock = mock.MagicMock() methodmock.return_value = jsonify({}) with mock.patch.dict(self._service.view_functions, {'WorkqueueService.get_output': methodmock}): response = self._inst.output(1) self.assertTrue(methodmock.called) self.assertEqual(response, {})
def test_remove(self): args = {'siteid': 12, 'filepath': '/data/somefile'} removemock = mock.MagicMock() removemock.return_value = jsonify(args) with mock.patch.dict(self._service.view_functions, {'WorkqueueService.remove': removemock}): response = self._inst.remove(**args) self.assertTrue(removemock.called) self.assertIsInstance(response, dict) self.assertEqual(response, args)
def get_output(job_id): """Get job output.""" Job = request.db.tables.Job # pylint: disable=invalid-name job = Job.query.filter_by(id=job_id, user_id=HRService.check_token())\ .first_or_404() log_filebase = os.path.join(current_app.workqueueservice_workerlogs, job.log_uid[:2], job.log_uid) elements_list = [] for element in job.elements: # pylint: disable=no-member attempt_output = { 'jobid': job_id, 'elementid': element.id, 'type': JobType(element.type).name } attempt_list = [] attempts = element.attempts element_log_filebase = os.path.join(log_filebase, str(element.id)) for attempt in xrange( 1, attempts): # previous attempts, presumably failed ones failed_output = dict(attempt_output, attempt=attempt, status=JobStatus.FAILED.name) log_filename = os.path.join(element_log_filebase, "attempt%i.log" % attempt) log = "log directory/file %s not found for job.element %s.%s."\ % (log_filename, job_id, element.id) if os.path.exists(log_filename): with open(log_filename, 'rb') as logfile: log = logfile.read() failed_output.update(log=log) attempt_list.append(failed_output) if attempts: status = JobStatus.FAILED if element.status == JobStatus.DONE: status = JobStatus.DONE last_output = dict(attempt_output, attempt=attempts, status=status.name) log_filename = os.path.join(element_log_filebase, "attempt%i.log" % attempts) log = "log directory/file %s not found for job.element %s.%s."\ % (log_filename, job_id, element.id) if os.path.exists(log_filename): with open(log_filename, 'rb') as logfile: log = logfile.read() last_output.update(log=log) if status == JobStatus.DONE and element.type == JobType.LIST: last_output.update(listing=element.listing) attempt_list.append(last_output) elements_list.append(attempt_list) return jsonify(elements_list)
def test_run(self): workload = [{ 'id': 1, 'user_id': 9, 'type': JobType.LIST, 'status': JobStatus.SUBMITTED, 'priority': 5, 'protocol': JobProtocol.DUMMY, 'src_siteid': 12, 'src_filepath': '/data/somefile', 'src_credentials': 'somesecret', 'dst_credentials': 'someothersecret', 'extra_opts': {}, 'elements': [{ "id": 0, "job_id": 1, "type": JobType.LIST, "src_filepath": "/some/file", "token": 'secret_token' }] }] getjobmock = mock.MagicMock() outputmock = mock.MagicMock() getjobmock.return_value = jsonify(workload) outputmock.return_value = '', 200 with mock.patch.dict(self._service.view_functions, {'WorkqueueService.get_next_job': getjobmock, 'WorkqueueService.return_output': outputmock}),\ mock.patch.object(self._inst._site_client, 'get_endpoints') as mock_get_endpoints,\ mock.patch('pdm.workqueue.Worker.X509Utils.add_ca_to_dir') as mock_ca2dir: mock_get_endpoints.return_value = { 'endpoints': ['blah1', 'blah2', 'blah3'], 'cas': ['blah blah', 'la la'] } mock_ca2dir.return_value = '/tmp/somecadir' self._inst.run() self.assertTrue(getjobmock.called) self.assertTrue(outputmock.called) self.assertTrue(mock_get_endpoints.called) self.assertEqual(mock_get_endpoints.call_count, 1) self.assertTrue(mock_ca2dir.called)
def test_copy(self): args = { 'src_siteid': 12, 'src_filepath': '/data/somefile', 'dst_siteid': 15, 'dst_filepath': '/data/someotherfile' } copymock = mock.MagicMock() copymock.return_value = jsonify(args) with mock.patch.dict(self._service.view_functions, {'WorkqueueService.copy': copymock}): response = self._inst.copy(**args) self.assertTrue(copymock.called) self.assertIsInstance(response, dict) self.assertEqual(response, args)
def get_jobs(): """Get all jobs for a user.""" Job = request.db.tables.Job # pylint: disable=invalid-name jobs = [] for job in Job.query.filter_by(user_id=HRService.check_token()).all(): elements = job.elements status_counter = Counter(element.status for element in elements) new_job = job.encode_for_json() new_job.update(num_elements=len(elements), num_new=status_counter[JobStatus.NEW], num_done=status_counter[JobStatus.DONE], num_failed=status_counter[JobStatus.FAILED], num_submitted=status_counter[JobStatus.SUBMITTED], num_running=status_counter[JobStatus.RUNNING]) jobs.append(new_job) return jsonify(jobs)
def get_session_info(site_id): """ Get the session info for the current user at a given site. """ log = current_app.log db = request.db Cred = db.tables.Cred user_id = SiteService.get_current_uid() cred = Cred.query.filter_by(cred_owner=user_id, site_id=site_id).first() res = {'ok': False} if cred: res['username'] = cred.cred_username res['expiry'] = cred.cred_expiry if cred.cred_expiry > datetime.datetime.utcnow(): res['ok'] = True log.info("Fetched info for user %u at site %u.", user_id, site_id) return jsonify(res)
def get_endpoints(site_id): """ Get a list of all endpoints at a given site_id. Designed for cert auth. """ db = request.db Site = db.tables.Site site = Site.query.filter_by(site_id=site_id).first_or_404() endpoints = [] for ep_info in site.endpoints: endpoints.append(ep_info.ep_uri) cas = [] if site.user_ca_cert: cas.append(site.user_ca_cert) if site.service_ca_cert: cas.append(site.service_ca_cert) # Build response dictionary res = {'endpoints': endpoints} if cas: res['cas'] = cas return jsonify(res)
def get_element_status(job_id, element_id): """Get element status.""" JobElement = request.db.tables.JobElement # pylint: disable=invalid-name element = JobElement.query.filter_by(id=element_id, job_id=job_id)\ .join(JobElement.job)\ .filter_by(user_id=HRService.check_token())\ .first_or_404() # pylint: disable=no-member monitoring_info = element.monitoring_info if monitoring_info is None: monitoring_info = {} return jsonify({ 'jobid': element.job_id, 'elementid': element.id, 'status': JobStatus(element.status).name, 'attempts': element.attempts, 'transferred': monitoring_info.get('transferred', 'N/A'), 'instant': monitoring_info.get('instant', 'N/A'), 'average': monitoring_info.get('average', 'N/A'), 'elapsed': monitoring_info.get('elapsed', 'N/A') })
def delete_user(): """ Delete a user. The user can only delete himself. :return: response object with code 200 if successful, 404 if the user does not exist """ user_id = HRService.check_token() # exception thrown if no user_id db = request.db User = request.db.tables.User user = User.query.filter_by(id=user_id).first() if not user: # Raise an HTTPException with a 404 not found status code HRService._logger.error( "GET: requested user for id %s doesn't exist ", user_id) abort(404) try: db.session.delete(user) current_app.site_client.set_token(request.raw_token) current_app.site_client.del_user(user_id) db.session.commit() HRService._logger.info(" User %s deleted successfully", user_id) except Exception: db.session.rollback() HRService._logger.error(" Failed to delete a user %s (%s)", user_id, sys.exc_info()) abort(500) response = jsonify([{ 'message': "user %s deleted successfully" % (user.email, ) }]) response.status_code = 200 return response
def get_next_job(): """Get the next job.""" current_app.log.debug("Worker requesting job batch, request: %s", pformat(request.data)) require_attrs('types') # Job = request.db.tables.Job # pylint: disable=invalid-name # JobElement = request.db.tables.JobElement # pylint: disable=invalid-name alg_name = request.data.get('algorithm', 'BY_NUMBER').upper() elements = Algorithm[alg_name]( **request.data.get('algorithm.args', {})) # elements = JobElement.query.filter(JobElement.status.in_((JobStatus.NEW, JobStatus.FAILED)), # JobElement.attempts < JobElement.max_tries)\ # .join(JobElement.job)\ # .filter(Job.type.in_(request.data['types']))\ # .order_by(Job.priority)\ # .order_by(Job.id)\ # .limit(10)\ # .all() if not elements: abort(404, description="No work to be done.") work = [] for job, elements_group in groupby(elements, key=attrgetter('job')): elements = [] # for element in list(elements_group): for element in iter(elements_group): element.status = JobStatus.SUBMITTED element.update() # should be a bulk update element_dict = element.asdict() element_dict['token'] = request.token_svc.issue( "%d.%d" % (job.id, element.id)) elements.append(element_dict) job.status = max(ele.status for ele in job.elements) job_dict = job.asdict() job_dict['elements'] = elements work.append(job_dict) job.update() current_app.log.debug("Sending worker job batch: %s", pformat(work)) return jsonify(work)
def get_user(): """ Get user own self based on user_id from the token passed in. :return: json response with user data or 404 if the user does not exist """ user_id = HRService.check_token() User = request.db.tables.User # user by id from the token user = User.query.filter_by(id=user_id).first() if not user: # Raise an HTTPException with a 404 not found status code HRService._logger.error( "GET: requested user for id %s doesn't exist ", user_id) abort(404) response = jsonify(user) response.status_code = 200 return response
def turtles_get(): """ Get a list of all turtle IDs and names. """ db = request.db Turtle = db.tables.Turtle res = {x.id: x.name for x in db.session.query(Turtle).all()} return jsonify(res)
def hello(): """ Return a test string. """ return jsonify("Hello World!\n")
def get_token(): """ Issue a token to the client. """ token = request.token_svc.issue("Hello") return jsonify(token)
def add_site(): """ Add a site in the database. """ log = current_app.log db = request.db Site = db.tables.Site Endpoint = db.tables.Endpoint site_data = {} endpoints = [] try: if not request.data: return "Missing POST data", 400 raw_site_data = json.loads(request.data) # Required fields for key in ('site_name', 'site_desc', 'auth_type', 'auth_uri', 'public', 'def_path'): raw_val = raw_site_data.get(key, None) if raw_val is None: return "Required field %s missing" % key, 400 site_data[key] = raw_val # Optional fields for key in ('user_ca_cert', 'service_ca_cert'): raw_val = raw_site_data.get(key, None) if raw_val: site_data[key] = raw_val # Check the auth types if site_data["auth_type"] not in (0, 1): log.warn("Unable to add site: Invalid auth_type (%s)", site_data["auth_type"]) return "Invalid auth_type.", 400 if not SiteService.check_uri(site_data["auth_uri"]): log.warn("Unable to add site: Invalid auth_uri (%s)", site_data["auth_uri"]) return "Invalid auth_uri.", 400 # Extra fields site_data["site_owner"] = SiteService.get_current_uid() # Endpoints raw_eps = raw_site_data.get('endpoints', []) for raw_ep in raw_eps: if not SiteService.check_uri(raw_ep): log.warn("Unable to add site: Bad endpoint (%s)", raw_ep) return "Invalid endpoint format.", 400 endpoints.extend(raw_eps) except Exception as err: log.warn("POST data error from client: %s", str(err)) return "Malformed POST data", 400 # Now actually try to create the site new_site = Site(**site_data) try: with managed_session(request) as session: session.add(new_site) session.flush() # Ensure new_site gets an ID site_id = new_site.site_id # Also create the endpoints for ep_uri in endpoints: session.add(Endpoint(site_id=site_id, ep_uri=ep_uri)) except IntegrityError: # site_name is probably not unique log.info("Failed to add new non-unique site %s.", new_site.site_name) return "site_name is not unique", 409 except Exception as err: # Some kind of other database error? log.error("Failed to add new site %s (%s).", site_data['site_name'], str(err)) return "Failed to add site to DB", 500 log.info("Added site %s with %u endpoints (ID %u).", new_site.site_name, len(endpoints), new_site.site_id) return jsonify(new_site.site_id)
def get_element_output(job_id, element_id, attempt=None): # pylint: disable=too-many-branches """Get job element output.""" Job = request.db.tables.Job # pylint: disable=invalid-name JobElement = request.db.tables.JobElement # pylint: disable=invalid-name log_uid, element = Job.query.with_entities(Job.log_uid, JobElement)\ .filter_by(id=job_id, user_id=HRService.check_token())\ .join(Job.elements)\ .filter_by(id=element_id)\ .first_or_404() log_filebase = os.path.join(current_app.workqueueservice_workerlogs, log_uid[:2], log_uid, str(element_id)) if element.attempts == 0: abort(404, description= "No attempts have yet been recorded for element %d of " "job %d. Please try later." % (element_id, job_id)) # pylint: disable=no-member attempt_output = { 'jobid': job_id, 'elementid': element_id, 'type': JobType(element.type).name } if attempt is not None: try: attempt = int( attempt ) # can't use the flask <int:attempt> converter with negatives except ValueError: abort(400, description="bad attempt, expected an integer.") if attempt < 0: attempt = element.attempts + attempt + 1 if attempt not in xrange(1, element.attempts + 1): abort( 404, description= "Invalid attempt '%s', job.element %s.%s has been tried %s " "time(s)" % (attempt, job_id, element_id, element.attempts)) log_filename = os.path.join(log_filebase, "attempt%i.log" % attempt) if not os.path.exists(log_filename): abort( 500, description= "log directory/file %s not found for job.element %s.%s." % (log_filename, job_id, element_id)) status = JobStatus.FAILED if attempt == element.attempts and element.status == JobStatus.DONE: status = JobStatus.DONE attempt_output.update(attempt=attempt, status=status.name) with open(log_filename, 'rb') as logfile: attempt_output.update(log=logfile.read()) if status == JobStatus.DONE and element.type == JobType.LIST: attempt_output.update(listing=element.listing) return jsonify(attempt_output) attempt_list = [] attempts = element.attempts for attempt_ in xrange( 1, attempts): # previous attempts, presumably failed ones failed_output = dict(attempt_output, attempt=attempt_, status=JobStatus.FAILED.name) log_filename = os.path.join(log_filebase, "attempt%i.log" % attempt_) log = "log directory/file %s not found for job.element %s.%s."\ % (log_filename, job_id, element.id) if os.path.exists(log_filename): with open(log_filename, 'rb') as logfile: log = logfile.read() failed_output.update(log=log) attempt_list.append(failed_output) if attempts: status = JobStatus.FAILED if element.status == JobStatus.DONE: status = JobStatus.DONE last_output = dict(attempt_output, attempt=attempts, status=status.name) log_filename = os.path.join(log_filebase, "attempt%i.log" % attempts) log = "log directory/file %s not found for job.element %s.%s."\ % (log_filename, job_id, element.id) if os.path.exists(log_filename): with open(log_filename, 'rb') as logfile: log = logfile.read() last_output.update(log=log) if status == JobStatus.DONE and element.type == JobType.LIST: last_output.update(listing=element.listing) attempt_list.append(last_output) return jsonify(attempt_list)
def get_job(job_id): """Get job.""" Job = request.db.tables.Job # pylint: disable=invalid-name job = Job.query.filter_by(id=job_id, user_id=HRService.check_token())\ .first_or_404() return jsonify(job)