def loadDatabaseFiles(self, mylog):
''' Load the static database files from Shock.
The static database files are stored in the directory specified by the
data_folder_path configuration variable. A file is only downloaded if
the file is not available on this system or the file has been updated
in Shock.
@param mylog Log object for messages
@return Nothing
@raise MissingFileError when database file is not found in Shock
'''
# Get the current info about the static database files from the cache file.
cacheFilename = self.StatusFiles['cache_file']
if os.path.exists(cacheFilename):
fileCache = json.load(open(cacheFilename, "r"))
else:
fileCache = dict()
# Create a shock client.
shockClient = ShockClient(self.shockURL)
# See if the static database files on this system are up-to-date with files stored in Shock.
shockFiles = dict(self.DataFiles.items() + self.SearchFiles.items())
for key in shockFiles:
# Get info about the file stored in Shock.
localPath = shockFiles[key]
name = os.path.basename(localPath)
nodelist = shockClient.query_node(
{'lookupname': 'ProbAnnoData/' + name})
if len(nodelist) == 0:
message = "Database file %s is not available from %s\n" % (
name, self.shockURL)
mylog.log_message(log.ERR, message) # MBM
raise MissingFileError(message)
node = nodelist[0]
# Download the file if the checksum does not match or the file is not available on this system.
download = False
if key in fileCache:
if node['file']['checksum']['md5'] != fileCache[key]['file'][
'checksum']['md5']:
download = True
else:
download = True
if os.path.exists(localPath) == False:
download = True
if download:
sys.stderr.write("Downloading %s to %s\n" % (key, localPath))
shockClient.download_to_path(node["id"], localPath)
fileCache[key] = node
mylog.log_message(log.INFO,
'Downloaded %s to %s' % (key, localPath))
# Save the updated cache file.
json.dump(fileCache, open(cacheFilename, "w"), indent=4)
return
def storeDatabaseFiles(self, token):
''' Store the static database files to Shock.
@param token: Authorization token for authenticating to shock
@return Nothing
'''
# Create a shock client.
shockClient = ShockClient(self.shockURL, token=token)
# Upload all of the static database files to shock.
fileCache = dict()
shockFiles = dict(self.DataFiles.items() + self.SearchFiles.items())
for key in shockFiles:
localPath = shockFiles[key]
name = os.path.basename(localPath)
if os.path.exists(localPath):
sys.stderr.write('Saving "%s"...' %(localPath))
# See if the file already exists in Shock.
query = { 'lookupname': 'ProbAnnoData/'+name }
nodelist = shockClient.query_node(query)
# Remove all instances of the file in Shock.
if nodelist != None:
for node in nodelist:
shockClient.delete_node(node['id'])
# Build the attributes for this file and store as json in a separate file.
moddate = time.ctime(os.path.getmtime(localPath))
attr = { 'lookupname': 'ProbAnnoData/'+name, 'moddate': moddate }
attrFilename = os.path.join(self.dataFolderPath, name+'.attr')
attrFid = open(attrFilename, 'w')
json.dump(attr, attrFid, indent=4)
attrFid.close()
# Upload the file to Shock.
metadata = shockClient.create_node(localPath, attrFilename)
fileCache[key] = metadata
os.remove(attrFilename)
# Remove the list of users from the read ACL to give the file public read permission.
# Note this needs to change for Shock version 0.9.5 but not sure how to set public ACLs.
readacl = shockClient.get_acl(metadata['id'])
shockClient.delete_acl(metadata['id'], 'read', readacl['read'][0])
sys.stderr.write('done\n')
else:
sys.stderr.write('Could not find "%s" so it was not saved\n' %(localPath))
# Save the metadata on all of the database files.
cacheFilename = os.path.join(self.dataFolderPath, StatusFiles['cache_file'])
json.dump(fileCache, open(cacheFilename, 'w'), indent=4)
return
def loadDatabaseFiles(self, mylog):
''' Load the static database files from Shock.
The static database files are stored in the directory specified by the
data_folder_path configuration variable. A file is only downloaded if
the file is not available on this system or the file has been updated
in Shock.
@param mylog Log object for messages
@return Nothing
@raise MissingFileError when database file is not found in Shock
'''
# Get the current info about the static database files from the cache file.
cacheFilename = self.StatusFiles['cache_file']
if os.path.exists(cacheFilename):
fileCache = json.load(open(cacheFilename, "r"))
else:
fileCache = dict()
# Create a shock client.
shockClient = ShockClient(self.shockURL)
# See if the static database files on this system are up-to-date with files stored in Shock.
shockFiles = dict(self.DataFiles.items() + self.SearchFiles.items())
for key in shockFiles:
# Get info about the file stored in Shock.
localPath = shockFiles[key]
name = os.path.basename(localPath)
nodelist = shockClient.query_node( { 'lookupname': 'ProbAnnoData/'+name } )
if len(nodelist) == 0:
message = "Database file %s is not available from %s\n" %(name, self.shockURL)
mylog.log_message(log.ERR, message) # MBM
raise MissingFileError(message)
node = nodelist[0]
# Download the file if the checksum does not match or the file is not available on this system.
download = False
if key in fileCache:
if node['file']['checksum']['md5'] != fileCache[key]['file']['checksum']['md5']:
download = True
else:
download = True
if os.path.exists(localPath) == False:
download = True
if download:
sys.stderr.write("Downloading %s to %s\n" %(key, localPath))
shockClient.download_to_path(node["id"], localPath)
fileCache[key] = node
mylog.log_message(log.INFO, 'Downloaded %s to %s' %(key, localPath))
# Save the updated cache file.
json.dump(fileCache, open(cacheFilename, "w"), indent=4)
return
def storeDatabaseFiles(self, token):
''' Store the static database files to Shock.
@param token: Authorization token for authenticating to shock
@return Nothing
'''
# Create a shock client.
shockClient = ShockClient(self.shockURL, token=token)
# Upload all of the static database files to shock.
fileCache = dict()
shockFiles = dict(self.DataFiles.items() + self.SearchFiles.items())
for key in shockFiles:
localPath = shockFiles[key]
name = os.path.basename(localPath)
if os.path.exists(localPath):
sys.stderr.write('Saving "%s"...' % (localPath))
# See if the file already exists in Shock.
query = {'lookupname': LOOKUP_NAME_PREFIX + '/' + name}
nodelist = shockClient.query_node(query)
# Remove all instances of the file in Shock.
if nodelist != None:
for node in nodelist:
shockClient.delete_node(node['id'])
# Build the attributes for this file and store as json in a separate file.
moddate = time.ctime(os.path.getmtime(localPath))
attr = {
'lookupname': LOOKUP_NAME_PREFIX + '/' + name,
'moddate': moddate
}
attrFilename = os.path.join(self.dataFolderPath,
name + '.attr')
attrFid = open(attrFilename, 'w')
json.dump(attr, attrFid, indent=4)
attrFid.close()
# Upload the file to Shock.
metadata = shockClient.create_node(localPath, attrFilename)
fileCache[key] = metadata
os.remove(attrFilename)
# Remove the list of users from the read ACL to give the file public read permission.
# Note this needs to change for Shock version 0.9.5 but not sure how to set public ACLs.
readacl = shockClient.get_acl(metadata['id'])
shockClient.delete_acl(metadata['id'], 'read',
readacl['read'][0])
sys.stderr.write('done\n')
else:
sys.stderr.write('Could not find "%s" so it was not saved\n' %
(localPath))
# Save the metadata on all of the database files.
cacheFilename = os.path.join(self.dataFolderPath,
self.StatusFiles['cache_file'])
json.dump(fileCache, open(cacheFilename, 'w'), indent=4)
return
