Python MultiWorker示例

示例#1

文件： besdirac-dms-dataset-get.py 项目： besdiracgrid/BESDIRAC

def datasetGet():
  badger = Badger()
  badger.updateDataset(setName)
  datasetTotalSize = badger.getDatasetMetadata(setName)['Value']['TotalSize']
  originLocalfileSize = getCurrentDirTotalSize(destDir) #check file size of the destDir before download
  print "start download..."
  start = time.time()

  dw = DownloadWorker()
  mw = MultiWorker(dw,5)
  mw.main()
  dw.Clear()

  total=time.time()-start
  print "Finished,total time is %s"%total

示例#2

文件： besdirac-dms-get-files.py 项目： besdiracgrid/BESDIRAC

      #print k

  def Do(self, item):
    badger = Badger()
    #print "world"
    result = badger.downloadFilesByFilelist([item])#,destDir)
    #print "result",result
    if result['OK']:
      self.db[item] = '2'
      self.db.sync()
  def Clear(self):
    transferOK = True
    for k,v in self.db.iteritems():
      if v=='0':
        transferOK = False
        print "Some files failed, you need run this script again"
        break
    self.db.close()
    if transferOK:
      print "All files transfer successful"
      os.remove(self.dbName)

dw = DownloadWorker()
mw = MultiWorker(dw,5)
mw.main()
dw.Clear()
total=time.time()-start
print "Finished,total time is %s"%total

exit(0)