def copy_global_tag(arguments): raise NotImplementedError("Copying Global Tags is currently not supported for this transition command-line interface for CondDBFW.") # set up connection source_connection = querying.connect(arguments.db, secrets=arguments.secrets, mode=arguments.mode, map_blobs=True) dest_connection = querying.connect(arguments.dest_db, secrets=arguments.secrets, mode=arguments.mode, map_blobs=True) # get CondDBFW Global Tag object global_tag = source_connection.global_tag(name=arguments.input_gt) if global_tag == None: raise Exception("Source Global Tag doesn't exist.") tag_names = global_tag.tags().get_members("tag_name").data() tags = source_connection.tag(name=tags) # copy global tag first global_tag.insertion_time = datetime.datetime.now() global_tag.validity = -1 dest_connection.write_and_commit(global_tag) for tag in tags: # create temporary argument class class args(object): def __init__(self): self.input_tag = tag.name self.dest_tag = tag.name self.start = 1 self.end = tag.latest_iov()+1 for attribute in dir(arguments): self.__dict__[attribute] = getattr(arguments, attribute) copy_tag(args())
def copy_tag(arguments): # set up connection source_connection = querying.connect(arguments.db, secrets=arguments.secrets, mode=arguments.mode, map_blobs=True) dest_connection = querying.connect(arguments.dest_db, secrets=arguments.secrets, mode=arguments.mode, map_blobs=True) # get tag from the source database, adjust it, and copy it (with the defined IOV range) to the destination database print("Reading source Tag.") source_tag = source_connection.tag(name=arguments.input_tag) if source_tag == None: raise Exception("Source Tag doesn't exist.") # get all IOVs within the range [start, end] print("Reading source IOVs.") since_range = source_connection.range(arguments.start, arguments.end) source_iovs = source_tag.iovs(since=since_range).data() # get hashes of all IOVs contained in the Tag in the source database print("Reading source Payloads.") hashes = source_tag.iovs().get_members("payload_hash").data() payloads = source_connection.payload(hash=hashes) print("Writing to destination database...") # set end_of_validity to -1 because sqlite doesn't support long ints source_tag.end_of_validity = -1 source_tag.name = arguments.dest_tag source_tag.modification_time = datetime.datetime.now() # create new iovs new_iovs = [] for iov in source_iovs: new_iovs.append(dest_connection.models["iov"](iov.as_dicts(convert_timestamps=False), convert_timestamps=False)) # write new tag to destination database print("Writing destination Tag.") if dest_connection.tag(name=arguments.dest_tag) != None: dest_connection.write_and_commit(source_tag) # write new iovs print("Writing IOVs to destination Tag.") for iov in new_iovs: if dest_connection.iov(tag_name=iov.tag_name, since=iov.since, insertion_time=iov.insertion_time) == None: dest_connection.write_and_commit(iov) # get payloads used by IOVs and copy those over print("Copying Payloads over.") for payload in payloads: if dest_connection.payload(hash=payload.hash) == None: dest_connection.write_and_commit(payload) print("Copy complete.")
def search(arguments): raise NotImplementedError("Todo") connection = querying.connect(arguments.db, secrets=arguments.secrets, mode=arguments.mode) search_string = connection.regexp(".*%s.*" % arguments.string)
def send_payloads(self, hashes, upload_session_id): """ Send a list of payloads corresponding to hashes we got from the SQLite file and filtered by asking the server. """ # if we have no hashes, we can't send anything # but don't exit since it might mean all the Payloads were already on the server if len(hashes) == 0: self._outputter.write( "No hashes to send - moving to metadata upload.") return True else: self._outputter.write("Sending payloads of hashes not found:") # construct connection string for local SQLite database file database = ( "sqlite://%s" % os.path.abspath(self.sqlite_file_name)) if type( self.sqlite_file_name) == str else self.sqlite_file_name # create CondDBFW connection that maps blobs - as we need to query for payload BLOBs (disabled by default in CondDBFW) self._outputter.write("\tConnecting to input SQLite database.") con = querying.connect(database, map_blobs=True) # query for the Payloads self._outputter.write( "\tGetting Payloads from SQLite database based on list of hashes." ) payloads = con.payload(hash=hashes) # if we get a single Payload back, put it in a list and turn it into a json_list if payloads.__class__ != data_sources.json_list: payloads = data_sources.json_data_node.make([payloads]) # close the session with the SQLite database file - we won't use it again con.close_session() # if found some Payloads, send them if payloads: # Note: there is an edge case in which the SQLite file could have been queried # to delete the Payloads since we queried it for IOV hashes. This may be handled in the next iteration. # send http post with data blob in body, and everything else as URL parameters # convert Payload to a dictionary - we can put most of this into the URL of the HTTPs request dicts = payloads.as_dicts() self._outputter.write("Uploading Payload BLOBs:") # for each payload, send the BLOB to the server for n, payload in enumerate(dicts): self._outputter.write( "\t(%d/%d) Sending payload with hash '%s'." % (n + 1, len(dicts), payload["hash"])) response = self.send_blob(payload, upload_session_id) # check response for errors no_error = self.check_response_for_error_key( response, exit_if_error=True) if not (no_error): return False self._outputter.write( "\tPayload sent - moving to next one.") self._outputter.write("All Payloads uploaded.") return True else: return False
def connect(connection_data=None): if connection_data == None: connection_data = { "db_alias": "orapro", "schema": "cms_conditions", "host": "oracle", "secrets": "/afs/cern.ch/cms/DB/conddb/.cms_cond/netrc" } connection = querying.connect(connection_data) return connection
def diff_of_gts(arguments): # get a CondDBFW Global Tag object for the first GT # then use the diff() method to draw the table of differences # set up connection connection = querying.connect(arguments.db, secrets=arguments.secrets, mode=arguments.mode) gt1 = connection.global_tag(name=arguments.gt1) gt2 = connection.global_tag(name=arguments.gt2) gt1.diff(gt2).as_table(columns=["Record", "Label", "%s Tag" % arguments.gt1, "%s Tag" % arguments.gt2])
def diff_of_tags(arguments): # get a CondDBFW Tag object for the first tag # then use the diff() method to draw the table of differences # set up connection connection = querying.connect(arguments.db, secrets=arguments.secrets, mode=arguments.mode) tag1 = connection.tag(name=arguments.tag1) tag2 = connection.tag(name=arguments.tag2) tag1.diff(tag2).as_table(columns=["since", arguments.tag1, arguments.tag2])
def connect(connection_data=None, mode="r", map_blobs=False, secrets=None, pooling=True): if connection_data == None: connection_data = "frontier://FrontierProd/CMS_CONDITIONS" connection = querying.connect(connection_data, mode=mode, map_blobs=map_blobs, secrets=secrets, pooling=pooling) connections.append(connection) return connection
def send_payloads(self, hashes, upload_session_id): """ Send a list of payloads corresponding to hashes we got from the SQLite file and filtered by asking the server. """ # if we have no hashes, we can't send anything # but don't exit since it might mean all the Payloads were already on the server if len(hashes) == 0: self._outputter.write("No hashes to send - moving to metadata upload.") return True else: self._outputter.write("Sending payloads of hashes not found:") # construct connection string for local SQLite database file database = ("sqlite://%s" % os.path.abspath(self.sqlite_file_name)) if isinstance(self.sqlite_file_name, str) else self.sqlite_file_name # create CondDBFW connection that maps blobs - as we need to query for payload BLOBs (disabled by default in CondDBFW) self._outputter.write("\tConnecting to input SQLite database.") con = querying.connect(database, map_blobs=True) # query for the Payloads self._outputter.write("\tGetting Payloads from SQLite database based on list of hashes.") payloads = con.payload(hash=hashes) # if we get a single Payload back, put it in a list and turn it into a json_list if payloads.__class__ != data_sources.json_list: payloads = data_sources.json_data_node.make([payloads]) # close the session with the SQLite database file - we won't use it again con.close_session() # if found some Payloads, send them if payloads: # Note: there is an edge case in which the SQLite file could have been queried # to delete the Payloads since we queried it for IOV hashes. This may be handled in the next iteration. # send http post with data blob in body, and everything else as URL parameters # convert Payload to a dictionary - we can put most of this into the URL of the HTTPs request dicts = payloads.as_dicts() self._outputter.write("Uploading Payload BLOBs:") # for each payload, send the BLOB to the server for n, payload in enumerate(dicts): self._outputter.write("\t(%d/%d) Sending payload with hash '%s'." % (n+1, len(dicts), payload["hash"])) response = self.send_blob(payload, upload_session_id) # check response for errors no_error = self.check_response_for_error_key(response, exit_if_error=True) if not(no_error): return False self._outputter.write("\tPayload sent - moving to next one.") self._outputter.write("All Payloads uploaded.") return True else: return False
def list_object(arguments): # set up connection connection = querying.connect(arguments.db, secrets=arguments.secrets, mode=arguments.mode) options = ["tag", "gt", "gts_for_tag"] number_of_options_given = 0 for option in options: if getattr(arguments, option): number_of_options_given += 1 if number_of_options_given != 1: print("You must specify a single object to list.") exit() if arguments.tag: tag_name = arguments.tag tag = connection.tag(name=tag_name) if tag: iovs = tag.iovs(amount=arguments.limit) iovs.as_table() else: print("The Tag '%s' was not found in the database '%s'." % (tag_name, arguments.db)) exit() elif arguments.gt: gt_name = arguments.gt gt = connection.global_tag(name=gt_name) if gt: gt_maps = gt.tags(amount=arguments.limit) gt_maps.as_table(hide=["global_tag_name"]) else: print("The Global Tag '%s' was not found in the database '%s'." % (gt_name, arguments.db)) exit() elif arguments.gts_for_tag: tag_name = arguments.gts_for_tag tag = connection.tag(name=tag_name) gts = tag.parent_global_tags(amount=arguments.limit) gts.as_table(columns=["name", "insertion_time", "snapshot_time"])
def connect(connection_data=None): if connection_data == None: connection_data = {"db_alias":"orapro", "schema" : "cms_conditions", "host":"oracle", "secrets":"/afs/cern.ch/cms/DB/conddb/.cms_cond/netrc"} connection = querying.connect(connection_data) return connection
def __init__(self, metadata_source=None, debug=False, verbose=False, testing=False, server="https://cms-conddb-dev.cern.ch/cmsDbCondUpload/", **kwargs): """ Upload constructor: Given an SQLite file and a Metadata sources, reads into a dictionary read for it to be encoded and uploaded. Note: kwargs is used to capture stray arguments - arguments that do not match keywords will not be used. Note: default value of service_url should be changed for production. """ # set private variables self._debug = debug self._verbose = verbose self._testing = testing # initialise server-side log data as empty string - will be replaced when we get a response back from the server self._log_data = "" self._SERVICE_URL = server self.upload_session_id = None # set up client-side log file self.upload_log_file_name = "upload_logs/upload_log_%d" % new_log_file_id() self._handle = open(self.upload_log_file_name, "a") # set up client-side logging object self._outputter = output(verbose=verbose, log_handle=self._handle) self._outputter.write("Using server instance at '%s'." % self._SERVICE_URL) # expect a CondDBFW data_source object for metadata_source if metadata_source == None: # no upload metadat has been given - we cannot continue with the upload self.exit_upload("A source of metadata must be given so CondDBFW knows how to upload conditions.") else: # set up global metadata source variable self.metadata_source = metadata_source.data() # check for the destination tag # this is required whatever type of upload we're performing if self.metadata_source.get("destinationTags") == None: self.exit_upload("No destination Tag was given.") else: if isinstance(self.metadata_source.get("destinationTags"), dict) and self.metadata_source.get("destinationTags").keys()[0] == None: self.exit_upload("No destination Tag was given.") # make sure a destination database was given if self.metadata_source.get("destinationDatabase") == None: self.exit_upload("No destination database was given.") # get Conditions metadata if self.metadata_source.get("sourceDB") == None and self.metadata_source.get("hashToUse") == None: """ If we have neither an sqlite file nor the command line data """ self.exit_upload("You must give either an SQLite database file, or the necessary command line arguments to replace one."\ + "\nSee --help for command line argument information.") elif self.metadata_source.get("sourceDB") != None: """ We've been given an SQLite file, so try to extract Conditions Metadata based on that and the Upload Metadata in metadata_source We now extract the Tag and IOV data from SQLite. It is added to the dictionary for sending over HTTPs later. """ # make sure we have an input tag to look for in the source db self.input_tag = metadata_source.data().get("inputTag") if self.input_tag == None: self.exit_upload("No input Tag name was given.") # set empty dictionary to contain Tag and IOV data from SQLite result_dictionary = {} self.sqlite_file_name = self.metadata_source["sourceDB"] if not(os.path.isfile(self.sqlite_file_name)): self.exit_upload("SQLite file '%s' given doesn't exist." % self.sqlite_file_name) sqlite_con = querying.connect("sqlite://%s" % os.path.abspath(self.sqlite_file_name)) self._outputter.write("Getting Tag and IOVs from SQLite database.") # query for Tag, check for existence, then convert to dictionary tag = sqlite_con.tag(name=self.input_tag) if tag == None: self.exit_upload("The source Tag '%s' you gave was not found in the SQLite file." % self.input_tag) tag = tag.as_dicts(convert_timestamps=True) # query for IOVs, check for existence, then convert to dictionaries iovs = sqlite_con.iov(tag_name=self.input_tag) if iovs == None: self.exit_upload("No IOVs found in the SQLite file given for Tag '%s'." % self.input_tag) iovs = iovs.as_dicts(convert_timestamps=True) iovs = [iovs] if not isinstance(iovs, list) else iovs """ Finally, get the list of all Payload hashes of IOVs, then compute the list of hashes for which there is no Payload for this is used later to decide if we can continue the upload if the Payload was not found on the server. """ iovs_for_hashes = sqlite_con.iov(tag_name=self.input_tag) if iovs_for_hashes.__class__ == data_sources.json_list: hashes_of_iovs = iovs_for_hashes.get_members("payload_hash").data() else: hashes_of_iovs = [iovs_for_hashes.payload_hash] self.hashes_with_no_local_payload = [payload_hash for payload_hash in hashes_of_iovs if sqlite_con.payload(hash=payload_hash) == None] # close session open on SQLite database file sqlite_con.close_session() elif metadata_source.data().get("hashToUse") != None: """ Assume we've been given metadata in the command line (since no sqlite file is there, and we have command line arguments). We now use Tag and IOV data from command line. It is added to the dictionary for sending over HTTPs later. """ # set empty dictionary to contain Tag and IOV data from command line result_dictionary = {} now = to_timestamp(datetime.now()) # tag dictionary will be taken from the server # this does not require any authentication tag = self.get_tag_dictionary() self.check_response_for_error_key(tag) iovs = [{"tag_name" : self.metadata_source["destinationTag"], "since" : self.metadata_source["since"], "payload_hash" : self.metadata_source["hashToUse"],\ "insertion_time" : now}] # hashToUse cannot be stored locally (no sqlite file is given), so register it as not found self.hashes_with_no_local_payload = [self.metadata_source["hashToUse"]] # Note: normal optimisations will still take place - since the hash checking stage can tell if hashToUse does not exist on the server side # if the source Tag is run-based, convert sinces to lumi-based sinces with lumi-section = 0 if tag["time_type"] == "Run": for (i, iov) in enumerate(iovs): iovs[i]["since"] = iovs[i]["since"] << 32 result_dictionary = {"inputTagData" : tag, "iovs" : iovs} # add command line arguments to dictionary # remembering that metadata_source is a json_dict object result_dictionary.update(metadata_source.data()) # store in instance variable self.data_to_send = result_dictionary # if the since doesn't exist, take the first since from the list of IOVs if result_dictionary.get("since") == None: result_dictionary["since"] = sorted(iovs, key=lambda iov : iov["since"])[0]["since"] elif self.data_to_send["inputTagData"]["time_type"] == "Run": # Tag time_type says IOVs use Runs for sinces, so we convert to Lumi-based for uniform processing self.data_to_send["since"] = self.data_to_send["since"] << 32 """ TODO - Settle on a single destination tag format. """ # look for deprecated metadata entries - give warnings # Note - we only really support this format try: if isinstance(result_dictionary["destinationTags"], dict): self._outputter.write("WARNING: Multiple destination tags in a single metadata source is deprecated.") except Exception as e: self._outputter.write("ERROR: %s" % str(e))
def __init__(self, metadata_source=None, debug=False, verbose=False, testing=False, server="https://cms-conddb-dev.cern.ch/cmsDbCondUpload/", **kwargs): """ Upload constructor: Given an SQLite file and a Metadata sources, reads into a dictionary read for it to be encoded and uploaded. Note: kwargs is used to capture stray arguments - arguments that do not match keywords will not be used. Note: default value of service_url should be changed for production. """ # set private variables self._debug = debug self._verbose = verbose self._testing = testing # initialise server-side log data as empty string - will be replaced when we get a response back from the server self._log_data = "" self._SERVICE_URL = server self.upload_session_id = None # set up client-side log file self.upload_log_file_name = "upload_logs/upload_log_%d" % new_log_file_id( ) self._handle = open(self.upload_log_file_name, "a") # set up client-side logging object self._outputter = output(verbose=verbose, log_handle=self._handle) self._outputter.write("Using server instance at '%s'." % self._SERVICE_URL) # expect a CondDBFW data_source object for metadata_source if metadata_source == None: # no upload metadat has been given - we cannot continue with the upload self.exit_upload( "A source of metadata must be given so CondDBFW knows how to upload conditions." ) else: # set up global metadata source variable self.metadata_source = metadata_source.data() # check for the destination tag # this is required whatever type of upload we're performing if self.metadata_source.get("destinationTags") == None: self.exit_upload("No destination Tag was given.") else: if type(self.metadata_source.get( "destinationTags")) == dict and self.metadata_source.get( "destinationTags").keys()[0] == None: self.exit_upload("No destination Tag was given.") # make sure a destination database was given if self.metadata_source.get("destinationDatabase") == None: self.exit_upload("No destination database was given.") # get Conditions metadata if self.metadata_source.get( "sourceDB") == None and self.metadata_source.get( "hashToUse") == None: """ If we have neither an sqlite file nor the command line data """ self.exit_upload("You must give either an SQLite database file, or the necessary command line arguments to replace one."\ + "\nSee --help for command line argument information.") elif self.metadata_source.get("sourceDB") != None: """ We've been given an SQLite file, so try to extract Conditions Metadata based on that and the Upload Metadata in metadata_source We now extract the Tag and IOV data from SQLite. It is added to the dictionary for sending over HTTPs later. """ # make sure we have an input tag to look for in the source db self.input_tag = metadata_source.data().get("inputTag") if self.input_tag == None: self.exit_upload("No input Tag name was given.") # set empty dictionary to contain Tag and IOV data from SQLite result_dictionary = {} self.sqlite_file_name = self.metadata_source["sourceDB"] if not (os.path.isfile(self.sqlite_file_name)): self.exit_upload("SQLite file '%s' given doesn't exist." % self.sqlite_file_name) sqlite_con = querying.connect( "sqlite://%s" % os.path.abspath(self.sqlite_file_name)) self._outputter.write("Getting Tag and IOVs from SQLite database.") # query for Tag, check for existence, then convert to dictionary tag = sqlite_con.tag(name=self.input_tag) if tag == None: self.exit_upload( "The source Tag '%s' you gave was not found in the SQLite file." % self.input_tag) tag = tag.as_dicts(convert_timestamps=True) # query for IOVs, check for existence, then convert to dictionaries iovs = sqlite_con.iov(tag_name=self.input_tag) if iovs == None: self.exit_upload( "No IOVs found in the SQLite file given for Tag '%s'." % self.input_tag) iovs = iovs.as_dicts(convert_timestamps=True) iovs = [iovs] if type(iovs) != list else iovs """ Finally, get the list of all Payload hashes of IOVs, then compute the list of hashes for which there is no Payload for this is used later to decide if we can continue the upload if the Payload was not found on the server. """ iovs_for_hashes = sqlite_con.iov(tag_name=self.input_tag) if iovs_for_hashes.__class__ == data_sources.json_list: hashes_of_iovs = iovs_for_hashes.get_members( "payload_hash").data() else: hashes_of_iovs = [iovs_for_hashes.payload_hash] self.hashes_with_no_local_payload = [ payload_hash for payload_hash in hashes_of_iovs if sqlite_con.payload(hash=payload_hash) == None ] # close session open on SQLite database file sqlite_con.close_session() elif metadata_source.data().get("hashToUse") != None: """ Assume we've been given metadata in the command line (since no sqlite file is there, and we have command line arguments). We now use Tag and IOV data from command line. It is added to the dictionary for sending over HTTPs later. """ # set empty dictionary to contain Tag and IOV data from command line result_dictionary = {} now = to_timestamp(datetime.now()) # tag dictionary will be taken from the server # this does not require any authentication tag = self.get_tag_dictionary() self.check_response_for_error_key(tag) iovs = [{"tag_name" : self.metadata_source["destinationTag"], "since" : self.metadata_source["since"], "payload_hash" : self.metadata_source["hashToUse"],\ "insertion_time" : now}] # hashToUse cannot be stored locally (no sqlite file is given), so register it as not found self.hashes_with_no_local_payload = [ self.metadata_source["hashToUse"] ] # Note: normal optimisations will still take place - since the hash checking stage can tell if hashToUse does not exist on the server side # if the source Tag is run-based, convert sinces to lumi-based sinces with lumi-section = 0 if tag["time_type"] == "Run": for (i, iov) in enumerate(iovs): iovs[i]["since"] = iovs[i]["since"] << 32 result_dictionary = {"inputTagData": tag, "iovs": iovs} # add command line arguments to dictionary # remembering that metadata_source is a json_dict object result_dictionary.update(metadata_source.data()) # store in instance variable self.data_to_send = result_dictionary # if the since doesn't exist, take the first since from the list of IOVs if result_dictionary.get("since") == None: result_dictionary["since"] = sorted( iovs, key=lambda iov: iov["since"])[0]["since"] elif self.data_to_send["inputTagData"]["time_type"] == "Run": # Tag time_type says IOVs use Runs for sinces, so we convert to Lumi-based for uniform processing self.data_to_send["since"] = self.data_to_send["since"] << 32 """ TODO - Settle on a single destination tag format. """ # look for deprecated metadata entries - give warnings # Note - we only really support this format try: if type(result_dictionary["destinationTags"]) == dict: self._outputter.write( "WARNING: Multiple destination tags in a single metadata source is deprecated." ) except Exception as e: self._outputter.write("ERROR: %s" % str(e))