def _integration(self, _db, id, name, protocol, goal): """Find or create the ExternalIntegration referred to.""" if not id and not name and not (protocol and goal): raise ValueError( "An integration must by identified by either ID, name, or the combination of protocol and goal." ) integration = None if id: integration = get_one( _db, ExternalIntegration, ExternalIntegration.id==id ) if not integration: raise ValueError("No integration with ID %s." % id) if name: integration = get_one(_db, ExternalIntegration, name=name) if not integration and not (protocol and goal): raise ValueError( 'No integration with name "%s". To create it, you must also provide protocol and goal.' % name ) if not integration and (protocol and goal): integration, is_new = get_one_or_create( _db, ExternalIntegration, protocol=protocol, goal=goal ) if name: integration.name = name return integration
def do_run(self, _db=None, cmd_args=None, output=sys.stdout): _db = _db or self._db args = self.parse_command_line(_db, cmd_args=cmd_args) if args.name: name = args.name integration = get_one(_db, ExternalIntegration, name=name) if not integration: integration = get_one(_db, ExternalIntegration, id=name) if integration: integrations = [integration] else: output.write( "Could not locate integration by name or ID: %s\n" % args ) integrations = [] else: integrations = _db.query(ExternalIntegration).order_by( ExternalIntegration.name, ExternalIntegration.id).all() if not integrations: output.write("No integrations found.\n") for integration in integrations: output.write( "\n".join( integration.explain(include_secrets=args.show_secrets) ) ) output.write("\n")
def to_collection(self, _db): """Find or create a Collection object for this Overdrive Advantage account. :return: a 2-tuple of Collections (primary Overdrive collection, Overdrive Advantage collection) """ # First find the parent Collection. parent = get_one(_db, Collection, external_account_id=self.parent_library_id, protocol=Collection.OVERDRIVE) if not parent: # Without the parent's credentials we can't access the child. raise ValueError( "Cannot create a Collection whose parent does not already exist." ) name = parent.name + " / " + self.name child, ignore = get_one_or_create(_db, Collection, parent_id=parent.id, protocol=Collection.OVERDRIVE, external_account_id=self.library_id, create_method_kwargs=dict(name=name)) # Set or update the name of the collection to reflect the name of # the library, just in case that name has changed. child.name = name return parent, child
def ensure_coverage(self, item, force=False): """Ensure coverage for one specific item. :param item: This should always be an Identifier, but this code will also work if it's an Edition. (The Edition's .primary_identifier will be covered.) :param force: Run the coverage code even if an existing coverage record for this item was created after `self.cutoff_time`. :return: Either a coverage record or a CoverageFailure. TODO: This could be abstracted and moved to BaseCoverageProvider. """ if isinstance(item, Identifier): identifier = item else: identifier = item.primary_identifier coverage_record = get_one( self._db, CoverageRecord, identifier=identifier, data_source=self.data_source, operation=self.operation, on_multiple='interchangeable', ) if not force and not self.should_update(coverage_record): return coverage_record counts, records = self.process_batch_and_handle_results( [identifier] ) if records: coverage_record = records[0] else: coverage_record = None return coverage_record
def collection(self): """Retrieve the Collection object associated with this Monitor. """ if not self.collection_id: return None return get_one(self._db, Collection, id=self.collection_id)
def ensure_coverage(self, item, force=False): """Ensure coverage for one specific item. TODO: Could potentially be moved into BaseCoverageProvider. :param force: Run the coverage code even if an existing coverage record for this item was created after `self.cutoff_time`. :return: Either a coverage record or a CoverageFailure. """ if isinstance(item, Identifier): identifier = item else: identifier = item.primary_identifier coverage_record = get_one( self._db, CoverageRecord, identifier=identifier, data_source=self.output_source, operation=self.operation, on_multiple='interchangeable', ) if not force and not self.should_update(coverage_record): return coverage_record counts, records = self.process_batch_and_handle_results([identifier]) if records: coverage_record = records[0] else: coverage_record = None return coverage_record
def update_work_for_edition(self, edition, even_if_no_author=False, immediately_presentation_ready=False): work = None # Find a pool for this edition. If we have CirculationData, a pool was created # when we imported the edition. If there was already a pool from a different data # source, that's fine too. pool = get_one(self._db, LicensePool, identifier=edition.primary_identifier) if pool: # Note: pool.calculate_work will call self.set_presentation_edition(), # which will find editions attached to same Identifier. work, is_new_work = pool.calculate_work( even_if_no_author=even_if_no_author) # Note: if pool.calculate_work found or made a work, it already called work.calculate_presentation() if work: if immediately_presentation_ready: # We want this book to be presentation-ready # immediately upon import. As long as no crucial # information is missing (like language or title), # this will do it. work.set_presentation_ready_based_on_content() return pool, work
def confirm(self, resource_id, secret): """Confirm a secret for a URI, or don't. :return: A Response containing a simple HTML document. """ if not secret: return self.html_response(404, _("No confirmation code provided")) if not resource_id: return self.html_response(404, _("No resource ID provided")) validation = get_one(self._db, Validation, secret=secret) resource = get_one(self._db, Resource, id=resource_id) if not resource: return self.html_response(404, _("No such resource")) if not validation: # The secret is invalid. This might be because the secret # is wrong, or because the Resource has already been # validated. # # Let's eliminate the 'Resource has already been validated' # possibility and take care of the other case next. if resource and resource.validation and resource.validation.success: return self.html_response( 200, _("This URI has already been validated.")) if (not validation or not validation.resource or validation.resource.id != resource_id): # For whatever reason the resource ID and secret don't match. # A generic error that doesn't reveal information is appropriate # in all cases. error = _("Confirmation code %r not found") % secret return self.html_response(404, error) # At this point we know that the resource has not been # confirmed, and that the secret matches the resource. The # only other problem might be that the validation has expired. if not validation.active: error = _( "Confirmation code %r has expired. Re-register to get another code." ) % secret return self.html_response(400, error) validation.mark_as_successful() resource = validation.resource message = _("You successfully confirmed %s.") % resource.href return self.html_response(200, message)
def _validated_at(self, hyperlink): validated_at = "Not validated" if hyperlink and hyperlink.resource: validation = get_one(self._db, Validation, resource=hyperlink.resource) if validation: return validation.started_at return validated_at
def libraries(self, library_name=None): """Find all libraries on which this script should operate. :param library_name: The library name passed in on the command line, if any. """ if library_name: library = get_one(self._db, Library, name=library_name) if not library: raise ValueError("No library with name %r" % library_name) return [library] return self.all_libraries
def do_run(self): existing_timestamp = get_one(self._db, Timestamp, service=self.name) if existing_timestamp: raise Exception( "Timestamp for Database Migration script already exists") migrations = self.fetch_migration_files()[0] most_recent_migration = self.sort_migrations(migrations)[-1] initial_timestamp = Timestamp.stamp(self._db, self.name) self.update_timestamp(initial_timestamp, most_recent_migration)
def test_success(self): script = ConfigureCollectionScript() l1, ignore = create( self._db, Library, name="Library 1", short_name="L1", ) l2, ignore = create( self._db, Library, name="Library 2", short_name="L2", ) l3, ignore = create( self._db, Library, name="Library 3", short_name="L3", ) self._db.commit() # Create a collection, set all its attributes, set a custom # setting, and associate it with two libraries. output = StringIO() script.do_run( self._db, ["--name=New Collection", "--protocol=Overdrive", "--library=L2", "--library=L1", "--setting=library_id=1234", "--external-account-id=acctid", "--url=url", "--username=username", "--password=password", ], output ) # The collection was created and configured properly. collection = get_one(self._db, Collection) eq_("New Collection", collection.name) eq_("url", collection.external_integration.url) eq_("acctid", collection.external_account_id) eq_("username", collection.external_integration.username) eq_("password", collection.external_integration.password) # Two libraries now have access to the collection. eq_([collection], l1.collections) eq_([collection], l2.collections) eq_([], l3.collections) # One CollectionSetting was set on the collection. [setting] = collection.external_integration.settings eq_("library_id", setting.key) eq_("1234", setting.value) # The output explains the collection settings. expect = ("Configuration settings stored.\n" + "\n".join(collection.explain()) + "\n") eq_(expect, output.getvalue())
def test_add_settings(self): script = ConfigureIntegrationScript() output = StringIO() script.do_run(self._db, [ "--protocol=aprotocol", "--goal=agoal", "--setting=akey=avalue", ], output) # An ExternalIntegration was created and configured. integration = get_one(self._db, ExternalIntegration, protocol="aprotocol", goal="agoal") expect_output = "Configuration settings stored.\n" + "\n".join( integration.explain()) + "\n" eq_(expect_output, output.getvalue())
def nlu_interface(nlu_inputs, sess, model): # nlu_output = {'nlu_result':{'intent':'', 'slots':[]}} # CONFIRM_LIST, REJECT_LIST = get_y_n() nlu_inputs = nlu_inputs.strip().replace(' ', '') assert type(nlu_inputs) == str inputs = cut_sentence(nlu_inputs) id_inputs = data_utils.nlu_input_to_token_ids(inputs, input_vocab_path, data_utils.tab_tokenizer) _inputs, _labels, _sequence_length = model.get_one([[id_inputs, [0]]], 0) # pdb.set_trace() _, step_loss, logits, summary = model.step(sess, _inputs, _labels, _sequence_length, True) label_logit = logits['label'][0] predict_label_ids = np.argmax(label_logit, 1)[:_sequence_length[0]] predict_label = [rev_label_vocab[x] for x in predict_label_ids] nlu_output = '\t'.join(predict_label) return nlu_output
def do_run(self): args = self.parse_command_line() last_run_date = args.last_run_date last_run_counter = args.last_run_counter existing_timestamp = get_one(self._db, Timestamp, service=self.name) if last_run_date: last_run_datetime = self.parse_time(last_run_date) if existing_timestamp: existing_timestamp.timestamp = last_run_datetime if last_run_counter: existing_timestamp.counter = last_run_counter else: existing_timestamp, ignore = get_one_or_create( self._db, Timestamp, service=self.name, timestamp=last_run_datetime ) if existing_timestamp: migrations, migrations_by_dir = self.fetch_migration_files() new_migrations = self.get_new_migrations( existing_timestamp, migrations ) if new_migrations: # Log the new migrations. print "%d new migrations found." % len(new_migrations) for migration in new_migrations: print " - %s" % migration self.run_migrations( new_migrations, migrations_by_dir, existing_timestamp ) else: print "No new migrations found. Your database is up-to-date." else: print "" print ( "NO TIMESTAMP FOUND. Run script with timestamp that indicates" " the last migration run against this database." ) self.arg_parser().print_help()
def initialize(cls, _db): """Find or create a local analytics service. """ # If a local analytics service already exists, return it. local_analytics = get_one(_db, ExternalIntegration, protocol=cls.__module__, goal=ExternalIntegration.ANALYTICS_GOAL) # If a local analytics service already exists, don't create a # default one. Otherwise, create it with default name of # "Local Analytics". if not local_analytics: local_analytics, ignore = create( _db, ExternalIntegration, protocol=cls.__module__, goal=ExternalIntegration.ANALYTICS_GOAL, name=str(cls.NAME)) return local_analytics
def run(self, cmd_args=None): parsed = self.parse_command_line(self._db, cmd_args) name = parsed.name authentication_url = parsed.authentication_url opds = parsed.opds web = parsed.web description = parsed.description aliases = parsed.alias places = parsed.place short_name = parsed.short_name shared_secret = parsed.shared_secret library, is_new = get_one_or_create( self._db, Library, authentication_url=authentication_url ) if name: library.name = name if opds: library.opds_url = opds if web: library.web_url = web if description: library.description = description if short_name: library.short_name = short_name if shared_secret: library.shared_secret = shared_secret if aliases: for alias in aliases: get_one_or_create(self._db, LibraryAlias, library=library, name=alias, language='eng') if places: for place_external_id in places: place = get_one(self._db, Place, external_id=place_external_id) get_one_or_create( self._db, ServiceArea, library=library, place=place ) self._db.commit()
def register(self, do_get=HTTP.debuggable_get): if flask.request.method == 'GET': document = self.registration_document return self.catalog_response(document) auth_url = flask.request.form.get("url") self.log.info("Got request to register %s", auth_url) if not auth_url: return NO_AUTH_URL integration_contact_uri = flask.request.form.get("contact") integration_contact_email = integration_contact_uri shared_secret = None auth_header = flask.request.headers.get('Authorization') if auth_header and isinstance( auth_header, basestring) and "bearer" in auth_header.lower(): shared_secret = auth_header.split(' ', 1)[1] self.log.info("Incoming shared secret: %s...", shared_secret[:4]) # If 'stage' is not provided, it means the client doesn't make the # testing/production distinction. We have to assume they want # production -- otherwise they wouldn't bother registering. library_stage = flask.request.form.get("stage") self.log.info("Incoming stage: %s", library_stage) library_stage = library_stage or Library.PRODUCTION_STAGE # NOTE: This is commented out until we can say that # registration requires providing a contact email and expect # every new library to be on a circulation manager that can meet # this requirement. # #integration_contact_email = self._required_email_address( # integration_contact_uri, # "Invalid or missing configuration contact email address" #) if isinstance(integration_contact_email, ProblemDetail): return integration_contact_email # Registration is a complex multi-step process. Start a subtransaction # so we can back out of the whole thing if any part of it fails. __transaction = self._db.begin_nested() library = None elevated_permissions = False if shared_secret: # Look up a library by the provided shared secret. This # will let us handle the case where the library has # changed URLs (auth_url does not match # library.authentication_url) but the shared secret is the # same. library = get_one(self._db, Library, shared_secret=shared_secret) if not library: __transaction.rollback() return AUTHENTICATION_FAILURE.detailed( _("Provided shared secret is invalid")) # This gives the requestor an elevated level of permissions. elevated_permissions = True library_is_new = False if library.authentication_url != auth_url: # The library's authentication URL has changed, # e.g. moved from HTTP to HTTPS. The registration # includes a valid shared secret, so it's okay to # modify the corresponding database field. # # We want to do this before the registration, so that # we request the new URL instead of the old one. library.authentication_url = auth_url if not library: # Either this is a library at a known authentication URL # or it's a brand new library. library, library_is_new = get_one_or_create( self._db, Library, authentication_url=auth_url) registrar = LibraryRegistrar(self._db, do_get=do_get) result = registrar.register(library, library_stage) if isinstance(result, ProblemDetail): __transaction.rollback() return result # At this point registration (or re-registration) has # succeeded, so we won't be rolling back the subtransaction # that created the Library. __transaction.commit() auth_document, hyperlinks_to_create = result # Now that we've completed the registration process, we # know the opds_url -- it's the 'start' link found in # the auth_document. # # Registration will fail if this link is missing or the # URL doesn't work, so we can assume this is valid. opds_url = auth_document.root['href'] if library_is_new: # The library was just created, so it had no opds_url. # Set it now. library.opds_url = opds_url # The registration process may have queued up a number of # Hyperlinks that needed to be created (taken from the # library's authentication document), but we also need to # create a hyperlink for the integration contact provided with # the registration request itself. if integration_contact_email: hyperlinks_to_create.append((Hyperlink.INTEGRATION_CONTACT_REL, [integration_contact_email])) reset_shared_secret = False if elevated_permissions: # If you have elevated permissions you may ask for the # shared secret to be reset. reset_shared_secret = flask.request.form.get( "reset_shared_secret", False) if library.opds_url != opds_url: # The library's OPDS URL has changed, e.g. moved from # HTTP to HTTPS. Since we have elevated permissions, # it's okay to modify the corresponding database # field. library.opds_url = opds_url for rel, candidates in hyperlinks_to_create: hyperlink, is_modified = library.set_hyperlink(rel, *candidates) if is_modified: # We need to send an email to this email address about # what just happened. This is either so the receipient # can confirm that the address works, or to inform # them a new library is using their address. try: hyperlink.notify(self.emailer, self.app.url_for) except SMTPException, e: # We were unable to send the email. return INTEGRATION_ERROR.detailed( _("SMTP error while sending email to %(address)s", address=hyperlink.resource.href))
def get_collection(major, minor): item = get_one(Conf.db, DisplayItem, beacon_major_id=int(major), beacon_minor_id=int(minor)) return flask.jsonify(**item.json)