def setUp(self): logger.warning('Testing API started...') os.environ['espa_api_testing'] = 'True' # create a user self.mock_user = MockUser() self.mock_order = MockOrder() user_id = self.mock_user.add_testing_user() order_id = self.mock_order.generate_testing_order(user_id) self.order = Order.find(order_id) self.user = User.find(user_id) self.product_id = 'LT05_L1TP_032028_20120425_20160830_01_T1' self.sensor_id = 'tm5_collection' self.staff_product_id = 'LE07_L1TP_010028_20050420_20160925_01_T1' self.staff_sensor = 'etm7_collection' self.global_product_id = 'LE70450302003206EDC01' staff_user_id = self.mock_user.add_testing_user() self.staff_user = User.find(staff_user_id) self.staff_user.update('is_staff', True) staff_order_id = self.mock_order.generate_testing_order(staff_user_id) staff_order = Order.find(staff_order_id) staff_scene = staff_order.scenes()[0] staff_scene.update('name', self.staff_product_id) user_scene = self.order.scenes()[0] user_scene.update('name', self.staff_product_id) with open(os.path.join(__location__, 'domain/restricted.yaml')) as f: self.restricted = yaml.load(f.read()) self.restricted['all']['role'].remove('restricted_prod')
def setUp(self): os.environ['espa_api_testing'] = 'True' # create a user self.mock_user = MockUser() self.mock_order = MockOrder() user_id = self.mock_user.add_testing_user() order_id = self.mock_order.generate_testing_order(user_id) self.order = Order.find(order_id) self.user = User.find(user_id) self.product_id = 'LT50150401987120XXX02' self.staff_product_id = 'LE70450302003206EDC01' staff_user_id = self.mock_user.add_testing_user() self.staff_user = User.find(staff_user_id) self.staff_user.update('is_staff', True) staff_order_id = self.mock_order.generate_testing_order(staff_user_id) staff_order = Order.find(staff_order_id) staff_scene = staff_order.scenes()[0] staff_scene.update('name', self.staff_product_id) user_scene = self.order.scenes()[0] user_scene.update('name', self.staff_product_id) with open('api/domain/restricted.yaml') as f: self.restricted = yaml.load(f.read()) self.restricted['all']['role'].remove('restricted_prod')
def verify_user(username, password): try: # usernames with spaces are valid in EE, though they can't be used for cache keys cache_key = '{}-credentials'.format(username.replace(' ', '_espa_cred_insert_')) cache_entry = cache.get(cache_key) if cache_entry: # Need to be encrypted? if cache_entry['password'] == password: user_entry = cache_entry['user_entry'] # User may have changed their password while it was still cached else: user_entry = User.get(username, password) else: user_entry = User.get(username, password) cache_entry = {'password': password, 'user_entry': user_entry} cache.set(cache_key, cache_entry, 7200) user = User(*user_entry) flask.g.user = user # Replace usage with cached version except Exception: logger.info('Invalid login attempt, username: {}'.format(username)) return False return True
def verify_user(username, password): if (username is None) or (not (str(username).strip())): logger.warning('Invalid username supplied: %s', username) flask.g.error_reason = 'auth' return False try: # usernames with spaces are valid in EE, though they can't be used for cache keys cache_key = '{}-credentials'.format( username.replace(' ', '_espa_cred_insert_')) cache_entry = cache.get(cache_key) if cache_entry: # Need to be encrypted? if cache_entry['password'] == password: user_entry = cache_entry['user_entry'] # User may have changed their password while it was still cached else: user_entry = User.get(username, password) else: user_entry = User.get(username, password) cache_entry = {'password': password, 'user_entry': user_entry} cache.set(cache_key, cache_entry, 7200) user = User(*user_entry) flask.g.user = user # Replace usage with cached version except UserException as e: logger.info('Invalid login attempt, username: {}, {}'.format( username, e)) flask.g.error_reason = 'unknown' return False except ERSApiAuthFailedException as e: logger.info('Invalid login attempt, username: {}, {}'.format( username, e)) flask.g.error_reason = 'auth' return False except ERSApiErrorException as e: logger.info('ERS lookup failed, username: {}, {}'.format(username, e)) flask.g.error_reason = 'unknown' return False except ERSApiConnectionException as e: logger.info('ERS is down {}'.format(e)) flask.g.error_reason = 'conn' return False except DBConnectException as e: logger.critical('! Database reported a problem: {}'.format(e)) flask.g.error_reasons = 'db' return False except Exception: logger.info('Invalid login attempt, username: {}'.format(username)) flask.g.error_reason = 'unknown' return False return True
def setUp(self): os.environ['espa_api_testing'] = 'True' # create a user self.mock_user = MockUser() self.mock_order = MockOrder() self.user = User.find(self.mock_user.add_testing_user()) self.order_id = self.mock_order.generate_testing_order(self.user.id) self.app = http.app.test_client() self.app.testing = True self.sceneids = self.mock_order.scene_names_list(self.order_id)[0:2] token = ''.format(self.user.username, 'foo') auth_string = "Basic {}".format(base64.b64encode(token)) self.headers = {"Authorization": auth_string} with db_instance() as db: uidsql = "select user_id, orderid from ordering_order limit 1;" db.select(uidsql) self.userid = db[0]['user_id'] self.orderid = db[0]['orderid'] itemsql = "select name, order_id from ordering_scene limit 1;" db.select(itemsql) self.itemid = db[0][0] itemorderid = db[0][1] ordersql = "select orderid from ordering_order where id = {};".format(itemorderid) db.select(ordersql) self.itemorderid = db[0][0] self.base_order = lowercase_all(testorders.build_base_order())
def generate_ee_testing_order(self, user_id, partial=False): ee_order = mock_lta.get_available_orders_partial(partial) # Have to emulate a bunch of load_ee_orders for eeorder, email_addr, contactid in ee_order: order_id = Order.generate_ee_order_id(email_addr, eeorder) scene_info = ee_order[eeorder, email_addr, contactid] user = User.find(user_id) ts = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S.%f') order_dict = { 'orderid': order_id, 'user_id': user.id, 'order_type': 'level2_ondemand', 'status': 'ordered', 'note': 'EarthExplorer order id: {}'.format(eeorder), 'ee_order_id': eeorder, 'order_source': 'ee', 'order_date': ts, 'priority': 'normal', 'email': user.email, 'product_options': 'include_sr: true', 'product_opts': Order.get_default_ee_options(scene_info) } order = Order.create(order_dict) self.production_provider.load_ee_scenes(scene_info, order.id) return order.id
def fetch_user_orders(self, username='', email='', user_id='', filters=None): if filters and not isinstance(filters, dict): raise OrderingProviderException('filters must be dict') if username: usearch = {'username': username} elif email: usearch = {'email': email} elif user_id: usearch = {'id': user_id} user = User.where(usearch) if len(user) != 1: return list() else: user = user.pop() if filters: params = dict(filters) params.update({'user_id': user.id}) else: params = {'user_id': user.id} resp = Order.where(params) return resp
def item_status(self, orderid, itemid='ALL', username=None, filters=None): user = User.by_username(username) if not isinstance(filters, dict): if filters is None: filters = dict() else: raise TypeError('supplied filters invalid') if orderid: orders = Order.where({'orderid': orderid}) else: orders = Order.where({'user_id': user.id}) search = dict() if 'status' in filters: search.update(status=(filters.get('status'), )) if 'name' in filters: search.update(name=(filters.get('name'), )) elif itemid is not "ALL": search.update(name=(itemid, )) response = dict() for order in orders: response[order.orderid] = order.scenes(search) return response
def generate_ee_testing_order(self, user_id, partial=False): ee_order = mock_lta.get_available_orders_partial(partial) # Have to emulate a bunch of load_ee_orders for eeorder, email_addr, contactid in ee_order: order_id = Order.generate_ee_order_id(email_addr, eeorder) scene_info = ee_order[eeorder, email_addr, contactid] user = User.find(user_id) ts = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S.%f') order_dict = {'orderid': order_id, 'user_id': user.id, 'order_type': 'level2_ondemand', 'status': 'ordered', 'note': 'EarthExplorer order id: {}'.format(eeorder), 'ee_order_id': eeorder, 'order_source': 'ee', 'order_date': ts, 'priority': 'normal', 'email': user.email, 'product_options': 'include_sr: true', 'product_opts': Order.get_default_ee_options(scene_info)} order = Order.create(order_dict) self.production_provider.load_ee_scenes(scene_info, order.id) return order.id
def setUp(self): os.environ['espa_api_testing'] = 'True' # create a user self.mock_user = MockUser() self.mock_order = MockOrder() self.user = User.find(self.mock_user.add_testing_user()) self.order_id = self.mock_order.generate_testing_order(self.user.id) self.app = http.app.test_client() self.app.testing = True self.sceneids = self.mock_order.scene_names_list(self.order_id)[0:2] token = ''.format(self.user.username, 'foo') auth_string = "Basic {}".format(base64.b64encode(token)) self.headers = {"Authorization": auth_string} with db_instance() as db: uidsql = "select user_id, orderid from ordering_order limit 1;" db.select(uidsql) self.userid = db[0]['user_id'] self.orderid = db[0]['orderid'] itemsql = "select name, order_id from ordering_scene limit 1;" db.select(itemsql) self.itemid = db[0][0] itemorderid = db[0][1] ordersql = "select orderid from ordering_order where id = {};".format( itemorderid) db.select(ordersql) self.itemorderid = db[0][0] self.base_order = lowercase_all(testorders.build_base_order())
def test_production_update_landsat_product_status(self): order = Order.find(self.mock_order.generate_testing_order(self.user_id)) for scene in order.scenes(): scene.status = 'submitted' scene.sensor_type = 'landsat' scene.save() self.assertTrue(production_provider.update_landsat_product_status(User.find(self.user_id).contactid))
def test_fetch_production_products_landsat(self): order_id = self.mock_order.generate_testing_order(self.user_id) # need scenes with statuses of 'processing' and 'ordered' self.mock_order.update_scenes(order_id, 'status', ['processing','oncache','ordered']) user = User.find(self.user_id) params = {'for_user': user.username, 'product_types': ['landsat']} response = api.fetch_production_products(params) self.assertTrue('bilbo' in response[0]['orderid'])
def generate_testing_order(self, user_id): user = User.find(user_id) # need to monkey with the email, otherwise we get collisions with each # test creating a new scratch order with the same user rand = str(random.randint(1, 99)) user.email = rand + user.email order = self.ordering_provider.place_order(self.base_order, user) return order.id
def setUp(self): os.environ['espa_api_testing'] = 'True' self.mock_user = MockUser() self.staffuser = User.find(self.mock_user.add_testing_user()) self.staffuser.update('is_staff', True) self.base_order = lowercase_all(testorders.build_base_order()) self.base_schema = BaseValidationSchema.request_schema
def generate_testing_order(self, user_id): user = User.find(user_id) # need to monkey with the email, otherwise we get collisions with each # test creating a new scratch order with the same user rand = str(random.randint(1, 99)) user.email = rand + user.email orderid = self.ordering_provider.place_order(self.base_order, user) order = Order.find(orderid) return order.id
def kill_user_jobs(self, username): _response = dict() try: for job_name in User.by_username(username).active_hadoop_job_names(): _response[job_name] = self.kill_job(self.job_names_ids()[job_name]) except AttributeError, e: if "object has no attribute 'active_hadoop_job_names'" in e.message: _response['msg'] = 'user not found' else: raise e
def test_fetch_production_products_landsat(self): os.environ['ESPA_M2M_MODE'] = 'LANDSAT,MODIS,URLS' order_id = self.mock_order.generate_testing_order(self.user_id) # need scenes with statuses of 'processing' self.mock_order.update_scenes(order_id, 'landsat', 'status', ['processing', 'oncache']) user = User.find(self.user_id) params = {'for_user': user.username, 'product_types': ['landsat']} response = api.fetch_production_products(params) self.assertTrue('bilbo' in response[0]['orderid']) os.environ['ESPA_M2M_MODE'] = ''
def test_fetch_production_products_sentinel(self): order_id = self.mock_order.generate_testing_order(self.user_id) # need scenes with statuses of 'processing' scenes = Scene.where({'order_id': order_id, 'sensor_type': 'sentinel'}) self.mock_order.update_scenes(order_id, 'sentinel', 'status', ['processing', 'oncache']) user = User.find(self.user_id) params = {'for_user': user.username, 'product_types': ['sentinel']} response = api.fetch_production_products(params) self.assertTrue('bilbo' in response[0]['orderid'])
def test_fetch_production_products_landsat(self): cfg.put('system.m2m_url_enabled', 'True') cfg.put('system.m2m_val_enabled', 'True') order_id = self.mock_order.generate_testing_order(self.user_id) # need scenes with statuses of 'processing' self.mock_order.update_scenes(order_id, 'landsat', 'status', ['processing', 'oncache']) user = User.find(self.user_id) params = {'for_user': user.username, 'product_types': ['landsat']} response = api.fetch_production_products(params) self.assertTrue('bilbo' in response[0]['orderid']) cfg.put('system.m2m_url_enabled', 'False') cfg.put('system.m2m_val_enabled', 'False')
def fetch_user_orders(self, uid, filters=None): # deal with unicode uid if isinstance(uid, basestring): uid = str(uid) try: user = User.where({'username': uid}).pop() except IndexError: try: user = User.where({'email': uid}).pop() except IndexError: return {'msg': 'sorry, no user matched {0}'.format(uid)} if filters and not isinstance(filters, dict): raise OrderingProviderException('filters param must be of type dict') elif filters: params = dict(filters) params.update({'user_id': user.id}) else: params = {'user_id': user.id} return {'orders': [o.orderid for o in Order.where(params)]}
def load_user(request): token = request.headers.get('Authorization') api_user = None if not token: token = request.args.get('token') if token: token = token.replace('Basic ', '', 1) try: token = base64.b64decode(token) except TypeError: pass username, password = token.split(":") # naive token user_entry = User.get(username, password) if user_entry: user = User(*user_entry) if user.id: api_user = user return api_user
def item_status(self, orderid, itemid='ALL', username=None): response = {} sql = "select oo.orderid, os.id scene_id, os.name, os.status, os.completion_date, os.note, " \ "os.product_dload_url, os.cksum_download_url, os.log_file_contents " \ "from ordering_order oo left join ordering_scene os on oo.id = " \ "os.order_id where oo.orderid = %s" user = User.by_username(username) if itemid is not "ALL": argtup = (orderid, itemid) sql += " AND os.name = %s;" else: argtup = (str(orderid)) sql += ";" with db_instance() as db: db.select(sql, argtup) items = [_ for _ in db.fetcharr] if items: id = items[0]['orderid'] response['orderid'] = {id: []} for item in items: try: ts = item['completion_date'].isoformat() except AttributeError: # completion_date not yet set ts = '' i = { 'scene_id': item['scene_id'], 'name': item['name'], 'status': item['status'], 'completion_date': ts, 'note': item['note'], 'product_dload_url': item['product_dload_url'], 'cksum_download_url': item['cksum_download_url'] } if user and user.is_staff(): i['log_file_contents'] = item['log_file_contents'] response['orderid'][id].append(i) else: response[ 'msg'] = 'sorry, no items matched orderid %s , itemid %s' % ( orderid, itemid) return response
def item_status(self, orderid, itemid='ALL', username=None): response = {} sql = "select oo.orderid, os.id scene_id, os.name, os.status, os.completion_date, os.note, " \ "os.product_dload_url, os.cksum_download_url, os.log_file_contents " \ "from ordering_order oo left join ordering_scene os on oo.id = " \ "os.order_id where oo.orderid = %s" user = User.by_username(username) if itemid is not "ALL": argtup = (orderid, itemid) sql += " AND os.name = %s;" else: argtup = (str(orderid)) sql += ";" with db_instance() as db: db.select(sql, argtup) items = [_ for _ in db.fetcharr] if items: id = items[0]['orderid'] response['orderid'] = {id: []} for item in items: try: ts = item['completion_date'].isoformat() except AttributeError: # completion_date not yet set ts = '' i = {'scene_id': item['scene_id'], 'name': item['name'], 'status': item['status'], 'completion_date': ts, 'note': item['note'], 'product_dload_url': item['product_dload_url'], 'cksum_download_url': item['cksum_download_url']} if user and user.is_staff(): i['log_file_contents'] = item['log_file_contents'] response['orderid'][id].append(i) else: response['msg'] = 'sorry, no items matched orderid %s , itemid %s' % (orderid, itemid) return response
def setUp(self): os.environ['espa_api_testing'] = 'True' # create a user self.mock_user = MockUser() self.mock_order = MockOrder() self.user = User.find(self.mock_user.add_testing_user()) self.order_id = self.mock_order.generate_testing_order(self.user.id) self.app = http.app.test_client() self.app.testing = True self.sceneids = self.mock_order.scene_names_list(self.order_id)[0:2] token = ':'.join((self.user.username, 'foo')) auth_string = "Basic {}".format(base64.b64encode(token)) self.headers = {"Authorization": auth_string} with db_instance() as db: uidsql = "select user_id, orderid from ordering_order limit 1;" db.select(uidsql) self.userid = db[0]['user_id'] self.orderid = db[0]['orderid'] itemsql = "select name, order_id from ordering_scene limit 1;" db.select(itemsql) self.itemid = db[0][0] itemorderid = db[0][1] ordersql = "select orderid from ordering_order where id = {};".format( itemorderid) db.select(ordersql) self.itemorderid = db[0][0] self.base_order = lowercase_all(testorders.build_base_order()) self.sensors = [ k for k in self.base_order.keys() if isinstance(self.base_order[k], dict) and 'inputs' in self.base_order[k] ] self.inputs = {s: self.base_order[s]['inputs'] for s in self.sensors} self.input_names_all = set([ i for sublist in [s for k, s in self.inputs.items()] for i in sublist ])
def get(version, email=None): filters = request.get_json(force=True, silent=True) search = dict(username=auth.username(), filters=filters) if email: # Allow user collaboration for usearch in ('email', 'username'): user = User.where({usearch: email}) if len(user): break if not len(user): response = MessagesResponse(warnings=["Username/email {} not found" .format(email)], code=200) return response() else: search = {'filters': filters, usearch: email} response = OrdersResponse(espa.fetch_user_orders(**search)) response.limit = ('orderid',) response.code = 200 return response()
def test_handle_orders_success(self): _ = self.mock_order.generate_testing_order(self.user_id) self.assertTrue( api.handle_orders({'username': User.find(self.user_id)}))
def contact_ids_list(self): users = User.where({'id >': 0}) return [u.contactid for u in users]
def get(cls, *args): user = User.where({'username': '******'})[0] return (user.username, user.email, user.first_name, user.last_name, user.contactid)
def available_products(self, product_id, username): """ Check to see what products are available to user based on an input list of scenes :param product_id: list of desired inputs :param username: username :return: dictionary """ user = User.by_username(username) pub_prods = copy.deepcopy(OrderingProvider.sensor_products(product_id)) with open(os.path.join(__location__, 'domain/restricted.yaml')) as f: restricted = yaml.safe_load(f.read()) role = False if user.is_staff() else True restrict_all = restricted.get('all', {}) all_role = restrict_all.get('role', []) all_by_date = restrict_all.get('by_date', {}) all_ordering_rsctd = restrict_all.get('ordering', []) upd = {'date_restricted': {}, 'ordering_restricted': {}} for sensor_type, prods in pub_prods.items(): if sensor_type == 'not_implemented': continue stype = sensor_type.replace( '_collection', '') if '_collection' in sensor_type else sensor_type sensor_restr = restricted.get(stype, {}) role_restr = sensor_restr.get('role', []) + all_role by_date_restr = sensor_restr.get('by_date', {}) # All overrides any sensor related dates by_date_restr.update(all_by_date) outs = pub_prods[sensor_type]['products'] ins = pub_prods[sensor_type]['inputs'] if sensor_type in all_ordering_rsctd: for sc_id in ins: if sensor_type in upd['ordering_restricted']: upd['ordering_restricted'][sensor_type].append(sc_id) else: upd['ordering_restricted'][sensor_type] = [sc_id] pub_prods.pop(sensor_type) continue remove_me = [] if role: for prod in role_restr: try: outs.remove(prod) except ValueError: continue for prod in outs: if prod in by_date_restr: r = sensor_restr['by_date'][prod] for sc_id in ins: obj = sensor.instance(sc_id) julian = '{}{}'.format(obj.year, obj.doy) if not julian_date_check(julian, r): remove_me.append(prod) if prod in upd['date_restricted']: upd['date_restricted'][prod].append(sc_id) else: upd['date_restricted'][prod] = [sc_id] for rem in remove_me: try: outs.remove(rem) except ValueError: continue if upd['date_restricted']: pub_prods.update(date_restricted=upd['date_restricted']) if upd['ordering_restricted']: pub_prods.update(ordering_restricted=upd['ordering_restricted']) return pub_prods
def add_testing_user(self): ''' add user record to test schemas auth_user table ''' user = User('bilbo_baggins', '*****@*****.**', 'bilbo', 'baggins', '123456') return user.id
def add_testing_user(self): ''' add user record to test schemas auth_user table ''' user = User.find_or_create_user('bilbo_baggins', '*****@*****.**', 'bilbo', 'baggins', '123456') return user
def available_products(self, product_id, username): """ Check to see what products are available to user based on an input list of scenes :param product_id: list of desired inputs :param username: username :return: dictionary """ user = User.by_username(username) pub_prods = copy.deepcopy(OrderingProvider.sensor_products(product_id)) with open('api/domain/restricted.yaml') as f: restricted = yaml.load(f.read()) role = False if user.is_staff() else True restrict_all = restricted.get('all', {}) all_role = restrict_all.get('role', []) all_by_date = restrict_all.get('by_date', {}) upd = {'date_restricted': {}} for sensor_type, prods in pub_prods.items(): if sensor_type == 'not_implemented': continue sensor_restr = restricted.get(sensor_type, {}) role_restr = sensor_restr.get('role', []) + all_role by_date_restr = sensor_restr.get('by_date', {}) # All overrides any sensor related dates by_date_restr.update(all_by_date) outs = pub_prods[sensor_type]['products'] ins = pub_prods[sensor_type]['inputs'] remove_me = [] if role: for prod in role_restr: try: outs.remove(prod) except ValueError: continue for prod in outs: if prod in by_date_restr: r = sensor_restr['by_date'][prod] for sc_id in ins: obj = sensor.instance(sc_id) julian = '{}{}'.format(obj.year, obj.doy) if not julian_date_check(julian, r): remove_me.append(prod) if prod in upd['date_restricted']: upd['date_restricted'][prod].append(sc_id) else: upd['date_restricted'][prod] = [sc_id] for rem in remove_me: try: outs.remove(rem) except ValueError: continue if upd['date_restricted']: pub_prods.update(upd) return pub_prods