def storeDatabaseFiles(self, token): ''' Store the static database files to Shock. @param token: Authorization token for authenticating to shock @return Nothing ''' # Create a shock client. shockClient = ShockClient(self.shockURL, token=token) # Upload all of the static database files to shock. fileCache = dict() shockFiles = dict(self.DataFiles.items() + self.SearchFiles.items()) for key in shockFiles: localPath = shockFiles[key] name = os.path.basename(localPath) if os.path.exists(localPath): sys.stderr.write('Saving "%s"...' %(localPath)) # See if the file already exists in Shock. query = { 'lookupname': 'ProbAnnoData/'+name } nodelist = shockClient.query_node(query) # Remove all instances of the file in Shock. if nodelist != None: for node in nodelist: shockClient.delete_node(node['id']) # Build the attributes for this file and store as json in a separate file. moddate = time.ctime(os.path.getmtime(localPath)) attr = { 'lookupname': 'ProbAnnoData/'+name, 'moddate': moddate } attrFilename = os.path.join(self.dataFolderPath, name+'.attr') attrFid = open(attrFilename, 'w') json.dump(attr, attrFid, indent=4) attrFid.close() # Upload the file to Shock. metadata = shockClient.create_node(localPath, attrFilename) fileCache[key] = metadata os.remove(attrFilename) # Remove the list of users from the read ACL to give the file public read permission. # Note this needs to change for Shock version 0.9.5 but not sure how to set public ACLs. readacl = shockClient.get_acl(metadata['id']) shockClient.delete_acl(metadata['id'], 'read', readacl['read'][0]) sys.stderr.write('done\n') else: sys.stderr.write('Could not find "%s" so it was not saved\n' %(localPath)) # Save the metadata on all of the database files. cacheFilename = os.path.join(self.dataFolderPath, StatusFiles['cache_file']) json.dump(fileCache, open(cacheFilename, 'w'), indent=4) return
def storeDatabaseFiles(self, token): ''' Store the static database files to Shock. @param token: Authorization token for authenticating to shock @return Nothing ''' # Create a shock client. shockClient = ShockClient(self.shockURL, token=token) # Upload all of the static database files to shock. fileCache = dict() shockFiles = dict(self.DataFiles.items() + self.SearchFiles.items()) for key in shockFiles: localPath = shockFiles[key] name = os.path.basename(localPath) if os.path.exists(localPath): sys.stderr.write('Saving "%s"...' % (localPath)) # See if the file already exists in Shock. query = {'lookupname': LOOKUP_NAME_PREFIX + '/' + name} nodelist = shockClient.query_node(query) # Remove all instances of the file in Shock. if nodelist != None: for node in nodelist: shockClient.delete_node(node['id']) # Build the attributes for this file and store as json in a separate file. moddate = time.ctime(os.path.getmtime(localPath)) attr = { 'lookupname': LOOKUP_NAME_PREFIX + '/' + name, 'moddate': moddate } attrFilename = os.path.join(self.dataFolderPath, name + '.attr') attrFid = open(attrFilename, 'w') json.dump(attr, attrFid, indent=4) attrFid.close() # Upload the file to Shock. metadata = shockClient.create_node(localPath, attrFilename) fileCache[key] = metadata os.remove(attrFilename) # Remove the list of users from the read ACL to give the file public read permission. # Note this needs to change for Shock version 0.9.5 but not sure how to set public ACLs. readacl = shockClient.get_acl(metadata['id']) shockClient.delete_acl(metadata['id'], 'read', readacl['read'][0]) sys.stderr.write('done\n') else: sys.stderr.write('Could not find "%s" so it was not saved\n' % (localPath)) # Save the metadata on all of the database files. cacheFilename = os.path.join(self.dataFolderPath, self.StatusFiles['cache_file']) json.dump(fileCache, open(cacheFilename, 'w'), indent=4) return
# Set the format based on the sequence file extension if the format argument was not specified. if args.format is None: if len(extensions) == 1: input['format'] = extensions.keys()[0] else: print "The format of the sequence files could not be determined. Set the format with the --format argument." exit(1) else: input['format'] = args.format # For each file, upload to shock (keep track of ids). for filename in fileList: print "Uploading sequence file '%s'" % (filename) node = shockClient.create_node(filename, '') input['node_ids'].append(node['id']) # Submit a job to build the distance matrix. try: jobid = cbdClient.build_matrix(input) except Exception as e: print 'Error starting job: ' + e.message if args.showError: traceback.print_exc(file=sys.stdout) # Delete all of the input files from shock if something went wrong. for nodeId in input['node_ids']: shockClient.delete_node(nodeId) exit(1) print "Job '%s' submitted" % (jobid) exit(0)
exit(1) # Show job info. if args.showTimes: print 'Job started at %s and finished at %s' % (info['started'], info['last_update']) # Create a shock client. shockClient = ShockClient(info['results']['shockurl'], ujsClient._headers['AUTHORIZATION']) # Download the output to the specified file and remove the file from shock. try: shockClient.download_to_path(info['results']['shocknodes'][0], args.outputPath) except Exception as e: print 'Error downloading distance matrix from %s: %s' % ( info['results']['shockurl'], e.message) traceback.print_exc(file=sys.stdout) try: shockClient.delete_node(info['results']['shocknodes'][0]) except Exception as e: print 'Error deleting distance matrix file from %s: ' % ( +info['results']['shockurl'], e.message) traceback.print_exc(file=sys.stdout) # Delete the job. ujsClient.delete_job(args.jobID) exit(0)
# Set the format based on the sequence file extension if the format argument was not specified. if args.format is None: if len(extensions) == 1: input['format'] = extensions.keys()[0] else: print "The format of the sequence files could not be determined. Set the format with the --format argument." exit(1) else: input['format'] = args.format # For each file, upload to shock (keep track of ids). for filename in fileList: print "Uploading sequence file '%s'" %(filename) node = shockClient.create_node(filename, '') input['node_ids'].append(node['id']) # Submit a job to build the distance matrix. try: jobid = cbdClient.build_matrix(input) except Exception as e: print 'Error starting job: '+e.message if args.showError: traceback.print_exc(file=sys.stdout) # Delete all of the input files from shock if something went wrong. for nodeId in input['node_ids']: shockClient.delete_node(nodeId) exit(1) print "Job '%s' submitted" %(jobid) exit(0)
# Check if the job is complete. if not info['complete']: print "Job '%s' has status '%s' and is working on task %s of %s. Check again later." \ %(args.jobID, info['status'], info['total_progress'], info['max_progress']) exit(1) # Show job info. if args.showTimes: print 'Job started at %s and finished at %s' %(info['started'], info['last_update']) # Create a shock client. shockClient = ShockClient(info['results']['shockurl'], ujsClient._headers['AUTHORIZATION']) # Download the output to the specified file and remove the file from shock. try: shockClient.download_to_path(info['results']['shocknodes'][0], args.outputPath) except Exception as e: print 'Error downloading distance matrix from %s: %s' %(info['results']['shockurl'], e.message) traceback.print_exc(file=sys.stdout) try: shockClient.delete_node(info['results']['shocknodes'][0]) except Exception as e: print 'Error deleting distance matrix file from %s: ' %(+info['results']['shockurl'], e.message) traceback.print_exc(file=sys.stdout) # Delete the job. ujsClient.delete_job(args.jobID) exit(0)