def storeDatabaseFiles(self, token):
''' Store the static database files to Shock.
@param token: Authorization token for authenticating to shock
@return Nothing
'''
# Create a shock client.
shockClient = ShockClient(self.shockURL, token=token)
# Upload all of the static database files to shock.
fileCache = dict()
shockFiles = dict(self.DataFiles.items() + self.SearchFiles.items())
for key in shockFiles:
localPath = shockFiles[key]
name = os.path.basename(localPath)
if os.path.exists(localPath):
sys.stderr.write('Saving "%s"...' %(localPath))
# See if the file already exists in Shock.
query = { 'lookupname': 'ProbAnnoData/'+name }
nodelist = shockClient.query_node(query)
# Remove all instances of the file in Shock.
if nodelist != None:
for node in nodelist:
shockClient.delete_node(node['id'])
# Build the attributes for this file and store as json in a separate file.
moddate = time.ctime(os.path.getmtime(localPath))
attr = { 'lookupname': 'ProbAnnoData/'+name, 'moddate': moddate }
attrFilename = os.path.join(self.dataFolderPath, name+'.attr')
attrFid = open(attrFilename, 'w')
json.dump(attr, attrFid, indent=4)
attrFid.close()
# Upload the file to Shock.
metadata = shockClient.create_node(localPath, attrFilename)
fileCache[key] = metadata
os.remove(attrFilename)
# Remove the list of users from the read ACL to give the file public read permission.
# Note this needs to change for Shock version 0.9.5 but not sure how to set public ACLs.
readacl = shockClient.get_acl(metadata['id'])
shockClient.delete_acl(metadata['id'], 'read', readacl['read'][0])
sys.stderr.write('done\n')
else:
sys.stderr.write('Could not find "%s" so it was not saved\n' %(localPath))
# Save the metadata on all of the database files.
cacheFilename = os.path.join(self.dataFolderPath, StatusFiles['cache_file'])
json.dump(fileCache, open(cacheFilename, 'w'), indent=4)
return
def storeDatabaseFiles(self, token):
''' Store the static database files to Shock.
@param token: Authorization token for authenticating to shock
@return Nothing
'''
# Create a shock client.
shockClient = ShockClient(self.shockURL, token=token)
# Upload all of the static database files to shock.
fileCache = dict()
shockFiles = dict(self.DataFiles.items() + self.SearchFiles.items())
for key in shockFiles:
localPath = shockFiles[key]
name = os.path.basename(localPath)
if os.path.exists(localPath):
sys.stderr.write('Saving "%s"...' % (localPath))
# See if the file already exists in Shock.
query = {'lookupname': LOOKUP_NAME_PREFIX + '/' + name}
nodelist = shockClient.query_node(query)
# Remove all instances of the file in Shock.
if nodelist != None:
for node in nodelist:
shockClient.delete_node(node['id'])
# Build the attributes for this file and store as json in a separate file.
moddate = time.ctime(os.path.getmtime(localPath))
attr = {
'lookupname': LOOKUP_NAME_PREFIX + '/' + name,
'moddate': moddate
}
attrFilename = os.path.join(self.dataFolderPath,
name + '.attr')
attrFid = open(attrFilename, 'w')
json.dump(attr, attrFid, indent=4)
attrFid.close()
# Upload the file to Shock.
metadata = shockClient.create_node(localPath, attrFilename)
fileCache[key] = metadata
os.remove(attrFilename)
# Remove the list of users from the read ACL to give the file public read permission.
# Note this needs to change for Shock version 0.9.5 but not sure how to set public ACLs.
readacl = shockClient.get_acl(metadata['id'])
shockClient.delete_acl(metadata['id'], 'read',
readacl['read'][0])
sys.stderr.write('done\n')
else:
sys.stderr.write('Could not find "%s" so it was not saved\n' %
(localPath))
# Save the metadata on all of the database files.
cacheFilename = os.path.join(self.dataFolderPath,
self.StatusFiles['cache_file'])
json.dump(fileCache, open(cacheFilename, 'w'), indent=4)
return
# Set the format based on the sequence file extension if the format argument was not specified.
if args.format is None:
if len(extensions) == 1:
input['format'] = extensions.keys()[0]
else:
print "The format of the sequence files could not be determined. Set the format with the --format argument."
exit(1)
else:
input['format'] = args.format
# For each file, upload to shock (keep track of ids).
for filename in fileList:
print "Uploading sequence file '%s'" % (filename)
node = shockClient.create_node(filename, '')
input['node_ids'].append(node['id'])
# Submit a job to build the distance matrix.
try:
jobid = cbdClient.build_matrix(input)
except Exception as e:
print 'Error starting job: ' + e.message
if args.showError:
traceback.print_exc(file=sys.stdout)
# Delete all of the input files from shock if something went wrong.
for nodeId in input['node_ids']:
shockClient.delete_node(nodeId)
exit(1)
print "Job '%s' submitted" % (jobid)
exit(0)
exit(1)
# Show job info.
if args.showTimes:
print 'Job started at %s and finished at %s' % (info['started'],
info['last_update'])
# Create a shock client.
shockClient = ShockClient(info['results']['shockurl'],
ujsClient._headers['AUTHORIZATION'])
# Download the output to the specified file and remove the file from shock.
try:
shockClient.download_to_path(info['results']['shocknodes'][0],
args.outputPath)
except Exception as e:
print 'Error downloading distance matrix from %s: %s' % (
info['results']['shockurl'], e.message)
traceback.print_exc(file=sys.stdout)
try:
shockClient.delete_node(info['results']['shocknodes'][0])
except Exception as e:
print 'Error deleting distance matrix file from %s: ' % (
+info['results']['shockurl'], e.message)
traceback.print_exc(file=sys.stdout)
# Delete the job.
ujsClient.delete_job(args.jobID)
exit(0)
# Set the format based on the sequence file extension if the format argument was not specified.
if args.format is None:
if len(extensions) == 1:
input['format'] = extensions.keys()[0]
else:
print "The format of the sequence files could not be determined. Set the format with the --format argument."
exit(1)
else:
input['format'] = args.format
# For each file, upload to shock (keep track of ids).
for filename in fileList:
print "Uploading sequence file '%s'" %(filename)
node = shockClient.create_node(filename, '')
input['node_ids'].append(node['id'])
# Submit a job to build the distance matrix.
try:
jobid = cbdClient.build_matrix(input)
except Exception as e:
print 'Error starting job: '+e.message
if args.showError:
traceback.print_exc(file=sys.stdout)
# Delete all of the input files from shock if something went wrong.
for nodeId in input['node_ids']:
shockClient.delete_node(nodeId)
exit(1)
print "Job '%s' submitted" %(jobid)
exit(0)
# Check if the job is complete.
if not info['complete']:
print "Job '%s' has status '%s' and is working on task %s of %s. Check again later." \
%(args.jobID, info['status'], info['total_progress'], info['max_progress'])
exit(1)
# Show job info.
if args.showTimes:
print 'Job started at %s and finished at %s' %(info['started'], info['last_update'])
# Create a shock client.
shockClient = ShockClient(info['results']['shockurl'], ujsClient._headers['AUTHORIZATION'])
# Download the output to the specified file and remove the file from shock.
try:
shockClient.download_to_path(info['results']['shocknodes'][0], args.outputPath)
except Exception as e:
print 'Error downloading distance matrix from %s: %s' %(info['results']['shockurl'], e.message)
traceback.print_exc(file=sys.stdout)
try:
shockClient.delete_node(info['results']['shocknodes'][0])
except Exception as e:
print 'Error deleting distance matrix file from %s: ' %(+info['results']['shockurl'], e.message)
traceback.print_exc(file=sys.stdout)
# Delete the job.
ujsClient.delete_job(args.jobID)
exit(0)
