def loadDatabaseFiles(self, mylog): ''' Load the static database files from Shock. The static database files are stored in the directory specified by the data_folder_path configuration variable. A file is only downloaded if the file is not available on this system or the file has been updated in Shock. @param mylog Log object for messages @return Nothing @raise MissingFileError when database file is not found in Shock ''' # Get the current info about the static database files from the cache file. cacheFilename = self.StatusFiles['cache_file'] if os.path.exists(cacheFilename): fileCache = json.load(open(cacheFilename, "r")) else: fileCache = dict() # Create a shock client. shockClient = ShockClient(self.shockURL) # See if the static database files on this system are up-to-date with files stored in Shock. shockFiles = dict(self.DataFiles.items() + self.SearchFiles.items()) for key in shockFiles: # Get info about the file stored in Shock. localPath = shockFiles[key] name = os.path.basename(localPath) nodelist = shockClient.query_node( {'lookupname': 'ProbAnnoData/' + name}) if len(nodelist) == 0: message = "Database file %s is not available from %s\n" % ( name, self.shockURL) mylog.log_message(log.ERR, message) # MBM raise MissingFileError(message) node = nodelist[0] # Download the file if the checksum does not match or the file is not available on this system. download = False if key in fileCache: if node['file']['checksum']['md5'] != fileCache[key]['file'][ 'checksum']['md5']: download = True else: download = True if os.path.exists(localPath) == False: download = True if download: sys.stderr.write("Downloading %s to %s\n" % (key, localPath)) shockClient.download_to_path(node["id"], localPath) fileCache[key] = node mylog.log_message(log.INFO, 'Downloaded %s to %s' % (key, localPath)) # Save the updated cache file. json.dump(fileCache, open(cacheFilename, "w"), indent=4) return
def storeDatabaseFiles(self, token): ''' Store the static database files to Shock. @param token: Authorization token for authenticating to shock @return Nothing ''' # Create a shock client. shockClient = ShockClient(self.shockURL, token=token) # Upload all of the static database files to shock. fileCache = dict() shockFiles = dict(self.DataFiles.items() + self.SearchFiles.items()) for key in shockFiles: localPath = shockFiles[key] name = os.path.basename(localPath) if os.path.exists(localPath): sys.stderr.write('Saving "%s"...' %(localPath)) # See if the file already exists in Shock. query = { 'lookupname': 'ProbAnnoData/'+name } nodelist = shockClient.query_node(query) # Remove all instances of the file in Shock. if nodelist != None: for node in nodelist: shockClient.delete_node(node['id']) # Build the attributes for this file and store as json in a separate file. moddate = time.ctime(os.path.getmtime(localPath)) attr = { 'lookupname': 'ProbAnnoData/'+name, 'moddate': moddate } attrFilename = os.path.join(self.dataFolderPath, name+'.attr') attrFid = open(attrFilename, 'w') json.dump(attr, attrFid, indent=4) attrFid.close() # Upload the file to Shock. metadata = shockClient.create_node(localPath, attrFilename) fileCache[key] = metadata os.remove(attrFilename) # Remove the list of users from the read ACL to give the file public read permission. # Note this needs to change for Shock version 0.9.5 but not sure how to set public ACLs. readacl = shockClient.get_acl(metadata['id']) shockClient.delete_acl(metadata['id'], 'read', readacl['read'][0]) sys.stderr.write('done\n') else: sys.stderr.write('Could not find "%s" so it was not saved\n' %(localPath)) # Save the metadata on all of the database files. cacheFilename = os.path.join(self.dataFolderPath, StatusFiles['cache_file']) json.dump(fileCache, open(cacheFilename, 'w'), indent=4) return
def loadDatabaseFiles(self, mylog): ''' Load the static database files from Shock. The static database files are stored in the directory specified by the data_folder_path configuration variable. A file is only downloaded if the file is not available on this system or the file has been updated in Shock. @param mylog Log object for messages @return Nothing @raise MissingFileError when database file is not found in Shock ''' # Get the current info about the static database files from the cache file. cacheFilename = self.StatusFiles['cache_file'] if os.path.exists(cacheFilename): fileCache = json.load(open(cacheFilename, "r")) else: fileCache = dict() # Create a shock client. shockClient = ShockClient(self.shockURL) # See if the static database files on this system are up-to-date with files stored in Shock. shockFiles = dict(self.DataFiles.items() + self.SearchFiles.items()) for key in shockFiles: # Get info about the file stored in Shock. localPath = shockFiles[key] name = os.path.basename(localPath) nodelist = shockClient.query_node( { 'lookupname': 'ProbAnnoData/'+name } ) if len(nodelist) == 0: message = "Database file %s is not available from %s\n" %(name, self.shockURL) mylog.log_message(log.ERR, message) # MBM raise MissingFileError(message) node = nodelist[0] # Download the file if the checksum does not match or the file is not available on this system. download = False if key in fileCache: if node['file']['checksum']['md5'] != fileCache[key]['file']['checksum']['md5']: download = True else: download = True if os.path.exists(localPath) == False: download = True if download: sys.stderr.write("Downloading %s to %s\n" %(key, localPath)) shockClient.download_to_path(node["id"], localPath) fileCache[key] = node mylog.log_message(log.INFO, 'Downloaded %s to %s' %(key, localPath)) # Save the updated cache file. json.dump(fileCache, open(cacheFilename, "w"), indent=4) return
def storeDatabaseFiles(self, token): ''' Store the static database files to Shock. @param token: Authorization token for authenticating to shock @return Nothing ''' # Create a shock client. shockClient = ShockClient(self.shockURL, token=token) # Upload all of the static database files to shock. fileCache = dict() shockFiles = dict(self.DataFiles.items() + self.SearchFiles.items()) for key in shockFiles: localPath = shockFiles[key] name = os.path.basename(localPath) if os.path.exists(localPath): sys.stderr.write('Saving "%s"...' % (localPath)) # See if the file already exists in Shock. query = {'lookupname': LOOKUP_NAME_PREFIX + '/' + name} nodelist = shockClient.query_node(query) # Remove all instances of the file in Shock. if nodelist != None: for node in nodelist: shockClient.delete_node(node['id']) # Build the attributes for this file and store as json in a separate file. moddate = time.ctime(os.path.getmtime(localPath)) attr = { 'lookupname': LOOKUP_NAME_PREFIX + '/' + name, 'moddate': moddate } attrFilename = os.path.join(self.dataFolderPath, name + '.attr') attrFid = open(attrFilename, 'w') json.dump(attr, attrFid, indent=4) attrFid.close() # Upload the file to Shock. metadata = shockClient.create_node(localPath, attrFilename) fileCache[key] = metadata os.remove(attrFilename) # Remove the list of users from the read ACL to give the file public read permission. # Note this needs to change for Shock version 0.9.5 but not sure how to set public ACLs. readacl = shockClient.get_acl(metadata['id']) shockClient.delete_acl(metadata['id'], 'read', readacl['read'][0]) sys.stderr.write('done\n') else: sys.stderr.write('Could not find "%s" so it was not saved\n' % (localPath)) # Save the metadata on all of the database files. cacheFilename = os.path.join(self.dataFolderPath, self.StatusFiles['cache_file']) json.dump(fileCache, open(cacheFilename, 'w'), indent=4) return