Beispiel #1
0
def train(args):
    check_present(
        args,
        ["train_corpus", "output_folder", "dev_corpus", "train_shuffle_seed"])

    train_corpus_path = args.train_corpus
    if args.train_shuffle_seed > 0:
        reader = sling.RecordReader(args.train_corpus)
        items = [(key, value) for key, value in reader]
        reader.close()
        r = random.Random(args.train_shuffle_seed)
        r.shuffle(items)
        train_corpus_path = os.path.join(args.output_folder,
                                         "train_shuffled.rec")
        writer = sling.RecordWriter(train_corpus_path)
        for key, value in items:
            writer.write(key, value)
        writer.close()
        print("Wrote shuffled train corpus to %s using seed %d" % \
              (train_corpus_path, args.train_shuffle_seed))

    # Setting an explicit seed for the sake of determinism.
    torch.manual_seed(1)

    # Make commons store if needed.
    if args.commons == '' or not os.path.exists(args.commons):
        if args.commons == '':
            fname = os.path.join(args.output_folder, "commons")
            print("Will create a commons store at", fname)
            args.commons = fname
        else:
            print("No commons found at", args.commons, ", creating it...")
        _, symbols = commons_builder.build(
            [train_corpus_path, args.dev_corpus], args.commons)
        print("Commons created at", args.commons, "with", len(symbols), \
            "symbols besides the usual ones.")

    # Make the training spec.
    spec = Spec()
    spec.build(args.commons, train_corpus_path)

    # Initialize the model with the spec and any word embeddings.
    caspar = Caspar(spec)
    embeddings_file = args.word_embeddings
    if embeddings_file == '': embeddings_file = None
    caspar.initialize(embeddings_file)

    tmp_folder = os.path.join(args.output_folder, "tmp")
    if not os.path.exists(tmp_folder):
        os.makedirs(tmp_folder)

    evaluator = partial(dev_accuracy, args.dev_corpus, tmp_folder)

    output_file_prefix = os.path.join(args.output_folder, "caspar")
    hyperparams = Hyperparams(args)
    print("Using hyperparameters:", hyperparams)

    trainer = Trainer(caspar, hyperparams, evaluator, output_file_prefix)
    train = Corpora(train_corpus_path, spec.commons, gold=True)
    trainer.train(train)
Beispiel #2
0
    def spec_i(self, i_mock):
        '''
        k, power/bispectrum (p0k, p2k, p4k, bk) of ith mock
        '''
        specdict = self.cat_corr['spec']

        # copy cat_corr dictionary 
        cat_corr_i = self.cat_corr.copy() 
        cat_corr_i['catalog']['n_mock'] = i_mock

        spec_i = Spec(self.type, cat_corr_i, **self.kwargs)
        spec_i.read() 

        if self.type == 'pk':
            spec_ell = ''.join(['p', str(specdict['ell']), 'k'])

            spec_i_spec = getattr(spec_i, spec_ell)

            return [spec_i.k, spec_i_spec]

        elif self.type == 'bk': 
            bk_i = getattr(spec_i, 'bk')
            qk_i = getattr(spec_i, 'qk')
            return [spec_i.k1, spec_i.k2, spec_i.k3, bk_i, qk_i]
        else: 
            raise NotImplementedError
Beispiel #3
0
def build_bk_wrapper(params): 
    """ Wrapper for calculating power/bispectrum
    """
    cat_corr = params[0]
    kwargs = {} 
    if len(params) > 2: 
        kwargs = params[1]

    spectrum = Spec('bk', cat_corr, **kwargs)
    print spectrum.file()
    spectrum.build()

    return None 
Beispiel #4
0
class _SpecPluginTestCase(PluginTester, unittest.TestCase):
    activate = '--with-spec'
    args = ['--no-spec-color']
    plugins = [Spec()]

    def _get_suitepath(self):
        return '_spec_test_cases/%s.py' % self.suitename

    suitepath = property(_get_suitepath)

    def assertContains(self, needle, haystack):
        assert needle in haystack,\
            "Failed to find:\n\n%s\ninside\n%s\n" % \
                (_prepend_in_each_line(needle), _prepend_in_each_line(haystack))

    def assertContainsInOutput(self, string):
        self.assertContains(string, six.text_type(self.output))

    def failIfContains(self, needle, haystack):
        assert needle not in haystack,\
            "Found:\n\n%s\ninside\n%s\n" % \
                (_prepend_in_each_line(needle), _prepend_in_each_line(haystack))

    def failIfContainsInOutput(self, string):
        self.failIfContains(string, six.text_type(self.output))
Beispiel #5
0
class TestPluginSpecWithFoobazAndStandardPluginsEnabled(
        TestPluginSpecWithFoobaz):
    plugins = [
        Spec(),
        nose.plugins.skip.Skip(),
        nose.plugins.deprecated.Deprecated()
    ]
Beispiel #6
0
    def __init__(self, *args, **kwargs):
        super(TopoSpec, self).__init__(*args, **kwargs)
        app_manager.require_app("ryu.topology.switches")
        CONF.register_opt(StrOpt("spec_path", default="spec/mininet.yml"))

        self.graph = nx.Graph()
        self.spec = Spec.from_yaml(CONF.spec_path)
        self._thread = hub.spawn_after(3, self._compare_spec)
Beispiel #7
0
    def main(self):
        """ Main function.
        Entry point of the program.
        """
        try:
            from spec import Spec
            self.setup_parser()
            args = self.parser.parse_args()

            self.log = get_logger()
            if args.verbose:
                self.log.setLevel('INFO')
            if args.debug:
                self.log.setLevel('DEBUG')

            pypi = Pypi2spec(args.package)
            pypi.retrieve_info()
            pypi.download()
            pypi.extract_sources()
            pypi.determine_arch()
            pypi.remove_sources()
            settings = Settings()
            spec = Spec(settings, pypi)
            spec.fill_spec_info()
            spec.get_template()
            spec.write_spec()
        except Pypi2specError, err:
            print err
Beispiel #8
0
def train(args):
  check_present(args, ["train_corpus", "output_folder", "dev_corpus"])

  # Setting an explicit seed for the sake of determinism.
  torch.manual_seed(1)

  # Make commons store if needed.
  if args.commons == '' or not os.path.exists(args.commons):
    if args.commons == '':
      fname = os.path.join(args.output_folder, "commons")
      print "Will create a commons store at", fname
      args.commons = fname
    else:
      print "No commons found at", args.commons, ", creating it..."
    _, symbols = commons_builder.build(
      [args.train_corpus, args.dev_corpus], args.commons)
    print "Commons created at", args.commons, "with", len(symbols), \
      "symbols besides the usual ones."

  # Make the training spec.
  spec = Spec()
  spec.build(args.commons, args.train_corpus)

  # Initialize the model with the spec and any word embeddings.
  caspar = Caspar(spec)
  embeddings_file = args.word_embeddings
  if embeddings_file == '': embeddings_file = None
  caspar.initialize(embeddings_file)

  tmp_folder = os.path.join(args.output_folder, "tmp")
  if not os.path.exists(tmp_folder):
    os.makedirs(tmp_folder)

  evaluator = partial(dev_accuracy,
                      args.commons,
                      args.dev_corpus,
                      tmp_folder)

  output_file_prefix = os.path.join(args.output_folder, "caspar")
  hyperparams = Hyperparams(args)
  print "Using hyperparameters:", hyperparams

  trainer = Trainer(caspar, hyperparams, evaluator, output_file_prefix)
  train = Corpora(args.train_corpus, spec.commons, gold=True)
  trainer.train(train)
Beispiel #9
0
    def generateVideo(self, scriptFile):
        with open(scriptFile) as t:
            self.rootSpec = Spec(yaml.safe_load(t), None)

        rootSpec = self.rootSpec
        self.framerate = rootSpec.get('framerate', 30)
        self.frameWidth = rootSpec.get('framewidth', 1440)
        self.frameHeight = rootSpec.get('frameheight', 1080)
        self.outputFrames = rootSpec.get('outputframes')
        self.limitFrames = rootSpec.get('limitframes')
        random.seed(rootSpec.get('randomseed'))

        outputFile = rootSpec.get('outputfile', 'video.mp4')
        videoOut = outputFile + '.temp.mp4'

        # Initialize data structures
        self.imageSpecQueue = queue.Queue()
        self.imageFrameQueue = queue.Queue()
        self.resultQueue = queue.Queue()
        self.prevSpec = None
        self.allImageSpecsInitialized = False

        # Prepare data structures for processing
        images = rootSpec.get('images', [])
        self.prepareImageSpecs(images, rootSpec)

        # Start one thread to initialize image specs
        threading.Thread(target=self.runnableInitImageSpecs).start()
        
        # Start processing image specs by launching worker threads
        self.globalFrameN = 0
        for _ in range(self.rootSpec.get('threads', 16)):
            threading.Thread(target=self.runnableProcessFrame).start()

        # In the current thread, wait for and write the results
        self.writer = imageio.get_writer(videoOut, 
            fps=self.framerate,
            macro_block_size=8)
        self.processResults()
        self.writer.close()

        # Join audio
        audioSpec = rootSpec.getSpec('audio')
        if not audioSpec is None:
            self.combineVideoWithAudio(audioSpec, videoOut, outputFile)
Beispiel #10
0
def run(args):
    check_present(args, ["input", "parser", "output"])
    assert os.path.exists(args.input), args.input
    assert os.path.exists(args.parser), args.parser

    # Read parser flow.
    flow = Flow()
    flow.load(args.parser)

    # Initialize the spec from the flow.
    spec = Spec()
    spec.from_flow(flow)

    # Initialize the model from the flow.
    caspar = Caspar(spec)
    caspar.from_flow(flow)

    corpus = Corpora(args.input, caspar.spec.commons)
    writer = sling.RecordWriter(args.output)
    count = 0
    for document in corpus:
        state, _, _, trace = caspar.forward(document,
                                            train=False,
                                            debug=args.trace)
        state.write()
        if trace:
            trace.write()
        writer.write(str(count), state.encoded())
        count += 1
        if count % 100 == 0:
            print "Annotated", count, "documents", now(), mem()
    writer.close()
    print "Annotated", count, "documents", now(), mem()
    print "Wrote annotated documents to", args.output

    if args.evaluate:
        f = tempfile.NamedTemporaryFile(delete=False)
        fname = f.name
        caspar.spec.commons.save(fname, binary=True)
        f.close()
        eval_result = frame_evaluation(gold_corpus_path=args.input, \
            test_corpus_path=args.output, commons=caspar.spec.commons)
        os.unlink(fname)
        return eval_result
Beispiel #11
0
def set_components(top):
    spec = Spec.get_spec('Iso8583 v0 - Test')
    top.title = top.title + ' - ' + spec.name
    spec_tree = ttk.Treeview(top, columns=('Field', 'Field Encoding', 'Field Value'), selectmode='browse')
    msg = spec.get_msg_by_name('Authorization Message - 0100')

    for field_def in msg.req_fields:
        print field_def.name
        spec_tree.insert('', 'end', field_def.name, text=field_def.name,
                         values=field_def.name + ',' + field_def.encoding)

    spec_tree.pack()
Beispiel #12
0
class TestPluginSpecWithDoctestsButDisabled(_SpecPluginTestCase):
    activate = '--with-spec'

    # no --spec-doctests option
    args = ['--with-doctest', '--doctest-tests', '--no-spec-color']
    plugins = [Spec(), nose.plugins.doctests.Doctest()]
    suitename = 'doctests'

    def test_doesnt_build_specifications_for_doctests_when_spec_doctests_option_wasnt_set(
            self):
        self.failIfContainsInOutput("test_doctests")
        self.failIfContainsInOutput("2 + 3 returns 5")
Beispiel #13
0
class TestPluginSpecWithDoctests(_SpecPluginTestCase):
    activate = '--with-spec'
    args = [
        '--with-doctest', '--doctest-tests', '--spec-doctests',
        '--no-spec-color'
    ]
    plugins = [Spec(), nose.plugins.doctests.Doctest()]

    suitename = 'doctests'
    expected_test_doctests_output = """doctests
- 2 + 3 returns 5
- None is nothing
- foobar throws "NameError: name 'foobar' is not defined"
"""

    def test_builds_specifications_for_doctests(self):
        self.assertContainsInOutput(self.expected_test_doctests_output)
Beispiel #14
0
machines = [Machine(1), Machine(2)]

# setup times dalla tabella 2 dell'articolo
setup_times = {
    "length": 60,
    "width": 15,
    "thickness": 10,
    "hardness": 20,
    "colour": 15,
}

# creazione delle specs per ogni job, dalla tabelle 2 dell'articolo
specs_list = {
    1: [
        Spec("length", 240, setup_times["length"]),
        Spec("width", 12, setup_times["width"]),
        Spec("thickness", 1.5, setup_times["thickness"]),
        Spec("hardness", 7, setup_times["hardness"]),
        Spec("colour", 1, setup_times["colour"])
    ],
    2: [
        Spec("length", 96, setup_times["length"]),
        Spec("width", 36, setup_times["width"]),
        Spec("thickness", 5.0, setup_times["thickness"]),
        Spec("hardness", 8, setup_times["hardness"]),
        Spec("colour", 1, setup_times["colour"])
    ],
    3: [
        Spec("length", 96, setup_times["length"]),
        Spec("width", 24, setup_times["width"]),
Beispiel #15
0
def set_components(top):
    spec = Spec.get_spec('Iso8583 v0 - Test')
    top.title = top.title + ' - ' + spec.name
    spec_tree = ttk.Treeview(top, columns=('Field', 'Field Encoding', 'Field Value'), selectmode='browse')
    msg = spec.get_msg_by_name('Authorization Message - 0100')

    for field_def in msg.req_fields:
        print field_def.name
        spec_tree.insert('', 'end', field_def.name, text=field_def.name,
                         values=field_def.name + ',' + field_def.encoding)

    spec_tree.pack()


if __name__ == "__main__":
    Spec.register_spec(Iso8583v0Spec())

    top = Tk.Tk()
    top.title = 'ISO8583 Simulator'
    top.configure(height=600, width=600)
    set_components(top)
    top.mainloop()


    # Spec.register_spec(Iso8583v0Spec())
    # spec = Spec.get_spec('Iso8583 v0 - Test')
    # msg = spec.parse(binascii.unhexlify(
    #    'f1f1f0f0' + 'F3000000200000018000000000000001' + '0F313231313131313131313131313134' + '303034303030' +
    #    '31323334353637383930' + '30333033' + '313130303030' + 'F0F1F5F1F2F1F1F1F1F1F1F1F1F1F1F1F1F4' +
    #    'e3456af57b0198f4' + 'F0F0F4F0F0F0F8' + 'e3456af57b0198f4'))
 def __init__(self):
     self.easy = int(input("Enter the marks for easy questions > "))
     self.medium = int(input("Enter the marks for medium questions > "))
     self.hard = int(input("Enter the marks for hard questions > "))
     self.entry = Spec(self.easy, self.medium, self.hard)
     self.store = QuestionStore()
Beispiel #17
0
class Cascade:
	checkTimeout = 10
	spec = None
	cascade = None
	
	lastPrice = None
	pricePrecision = None
	minAmount = None
	fee = None
	
	activeOrdersCount = 5
	totalPrecision = 8
	profitPrecision = 2
	pair = 'btc_usd'
	
	profitPercent = 1
	startPercent = 1
	deepPercent = 15
	totalInvest = 5
	
	allowRevers = False
	
	
	def setActiveOrdersCount(self, activeOrdersCount):
		self.activeOrdersCount = activeOrdersCount
	
	def setTotalPrecision(self, totalPrecision):
		self.totalPrecision = totalPrecision
	
	def setProfitPrecision(self, profitPrecision):
		self.profitPrecision = profitPrecision
	
	def setPair(self, pair):
		self.pair = pair
		if not self.spec.checkConnection():
			quit()
			
		if not self.spec.loadTradeConditions():
			quit()

		if not self.spec.loadTickers([pair]):
			quit()
		
		self.lastPrice = self.spec.tickers[pair]['last']
		self.pricePrecision = self.spec.pairs[pair]['decimal_places']
		self.minAmount = self.spec.pairs[pair]['min_amount']
		self.fee = self.spec.pairs[pair]['fee']

	def setProfitPercent(self, profitPercent):
		self.profitPercent = profitPercent
		
	def setStartPercent(self, startPercent):
		self.startPercent = startPercent
		
	def setDeepPercent(self, deepPercent):
		self.deepPercent = deepPercent
	
	def setTotalInvest(self, totalInvest):
		self.totalInvest = totalInvest
	
	def setAllowRevers(self, allowRevers):
		self.allowRevers = allowRevers
		
	def __init__(self, key = None, secret = None, silent = False):
		self.spec = Spec(key, secret, silent)
	
	def setCascade(self, cascade):
		self.cascade = cascade
	
	def getCascade(self):
		return self.cascade
	
	def printCascade(self, cascade = None):
		if (cascade is None):
			print('printCascade. Cascade element not defined')
			quit()
		
		if len(cascade) > 0 and 'options' in cascade[0]:
			returned = 0
			accepted = 0
			profit = 0
			
			for item in cascade:
				accepted += round(item['sellOrder']['price'] * item['sellOrder']['operationAmount'] * (100 - self.fee) / 100, self.pricePrecision)
				returned = round(item['buyOrder']['operationAmount'] * (100 - self.fee) / 100, self.totalPrecision)
				profit = round(accepted - item['buyOrder']['operationAmount'] * item['buyOrder']['price'], self.profitPrecision)
				
				print('{0[stage]:>3} sell {1[operationAmount]:<12}@ {1[price]:<12}acc: {3:<12} buy: {2[operationAmount]:<12}@ {2[price]:<12}ret: {4:<14} {5:<2}'.format(item, item['sellOrder'], item['buyOrder'], accepted, returned, profit))
		
			return
			
		invested = 0
		for item in cascade:
			invested += round(item['buyOrder']['price'] * item['buyOrder']['operationAmount'], self.totalPrecision)
			accepted = round(item['sellOrder']['price'] * item['sellOrder']['operationAmount'] * (100 - self.fee) / 100, self.totalPrecision)
			profit = round(accepted - invested, self.profitPrecision)
			print('{0[stage]:>3} buy {1[operationAmount]:<12}@ {1[price]:<12}inv: {3:<12} sell: {2[operationAmount]:<12}@ {2[price]:<12}accp: {4:<14} {5:<2}'.format(item, item['buyOrder'], item['sellOrder'], invested, accepted, profit))
		
	
	def createCascade(self, instrumentVolume = None, orderId = None, maxStages = 150):
		self.setPair(self.pair)
		
		if not instrumentVolume is None:
			instrumentVolume = float(instrumentVolume)
		self.totalInvest = float(self.totalInvest)
		
		if orderId:
			startPrice = round(self.lastPrice * (100 + self.startPercent) / 100, self.pricePrecision)
			endPrice = round(startPrice * (100 + self.deepPercent) / 100, self.pricePrecision)
			priceLength = endPrice - startPrice
			investFreq = round(instrumentVolume / priceLength, self.totalPrecision)
			investQuant = round(self.minAmount * (100 + 2 * self.fee) / 100, self.totalPrecision)
			priceStep = priceLength * investQuant / instrumentVolume
			#priceStep = round(priceLength * investQuant / instrumentVolume, self.pricePrecision)
			
			if priceStep < 10 ** -self.pricePrecision:
				priceStep = 10 ** -self.pricePrecision
				investQuant = round(priceStep * investFreq, self.pricePrecision)
			
			if instrumentVolume // investQuant > maxStages:
				investQuant = round(instrumentVolume / maxStages, self.totalPrecision)
				priceStep = priceLength / maxStages
				#priceStep = round(priceLength / maxStages, self.pricePrecision)
			else:
				maxStages = int(instrumentVolume // investQuant)
				investQuant = round(instrumentVolume / (instrumentVolume // investQuant), self.totalPrecision)
				priceStep = priceLength / (instrumentVolume // investQuant)
				#priceStep = round(priceLength / (instrumentVolume // investQuant), self.pricePrecision)
			
			acceptedSumm = 0
			cascade = []
			for stage in range(0, maxStages):
				curPrice = round(startPrice + priceStep * stage, self.pricePrecision)
				accepted = investQuant * (stage + 1)
				acceptedSumm += round(investQuant * curPrice * (100 - self.fee) / 100, self.pricePrecision)
				buyAmount = round(accepted * 100 / (100 - self.fee), self.totalPrecision)
				buyPrice = round(acceptedSumm / accepted * (100 - self.profitPercent) / 100, self.pricePrecision)
				cascade.append({
					'stage': stage, 
					'buyOrder': {'pair': self.pair, 'action': 'buy', 'price': buyPrice, 'operationAmount': buyAmount}, 
					'sellOrder': {'pair': self.pair, 'action': 'sell', 'price': curPrice, 'operationAmount': investQuant}, 
					'options' : {'order_id': orderId, 'amount': instrumentVolume}})
			return cascade
			

		startPrice = round(self.lastPrice * (100 - self.startPercent) / 100, self.pricePrecision)
		endPrice = round(startPrice * (100 - self.deepPercent) / 100, self.pricePrecision)
		priceLength = startPrice - endPrice
		investFreq = round((startPrice - endPrice) / self.totalInvest, self.totalPrecision)
		investQuant = round(startPrice * self.minAmount * (100 + 2 * self.fee) / 100, self.totalPrecision)
		priceStep = investQuant * investFreq
		#priceStep = round(investQuant * investFreq, self.pricePrecision)
		
		if investQuant * investFreq < 10 ** -self.pricePrecision:
			priceStep = 10 ** -self.pricePrecision
			investQuant = round(priceStep / investFreq, self.totalPrecision)
		
		if self.totalInvest // investQuant > maxStages:
			investQuant = round(self.totalInvest / maxStages, self.totalPrecision)
			priceStep = priceLength / maxStages
			#priceStep = round(priceLength / maxStages, self.pricePrecision)
		else:
			maxStages = int(self.totalInvest // investQuant)
			investQuant = round(self.totalInvest / (self.totalInvest // investQuant), self.totalPrecision)
			priceStep = priceLength / (self.totalInvest // investQuant)
			#priceStep = round(priceLength / (self.totalInvest // investQuant), self.pricePrecision)
			
		sellAmount = 0
		cascade = []
		for stage in range(0, maxStages):
			curPrice = round(startPrice - priceStep * stage, self.pricePrecision)
			curAmount = round(investQuant / curPrice, self.totalPrecision)
			if curAmount < self.minAmount:
				curAmount = self.minAmount
			invested = investQuant * (stage + 1)
			sellAmount += round(curAmount * (100 - self.fee) / 100, self.totalPrecision)
			sellPrice = round(invested / sellAmount * (100 + self.profitPercent) / 100, self.pricePrecision)
			cascade.append({'stage': stage, 'buyOrder': {'pair': self.pair, 'action': 'buy', 'price': curPrice, 'operationAmount': curAmount}, 'sellOrder': {'pair': self.pair, 'action': 'sell', 'price': sellPrice, 'operationAmount': sellAmount}})

		return cascade
		

	## 
	#  @brief Brief
	#  
	#  @param [in] self Parameter_Description
	#  @param [in] cascade Parameter_Description
	#  @return True if can start reverse cascade, False in other case
	#  
	#  @details 0 - active, 1 - complete, 2 - canceled
	#  		
	def needReverse(self, cascade = None):
		if (cascade is None):
			cascade = self.cascade
		
		if (cascade is None):
			return False
		if len(cascade) == 0:
			return False
		if 'options' in cascade[0]:
			return False
		if not self.allowRevers:
			return False
		
		idx = len(cascade) - 1
		if 'status' in cascade[idx]['buyOrder'] and 'status' in cascade[idx]['sellOrder']:
			if cascade[idx]['buyOrder']['status'] == 1 and cascade[idx]['sellOrder']['status'] == 0:
				if round(self.lastPrice * (100 + self.startPercent) / 100, self.pricePrecision) < cascade[idx]['sellOrder']['price']:
					return True
		
		return False
	
	## 
	#  @brief detect type of cascade True - reverse, False - Normal
	#  
	#  @param [in] self Parameter_Description
	#  @param [in] cascade Parameter_Description
	#  @return Return_Description
	#  
	#  @details what doing with incorrect cascade structure?
	#  	
	def isRevers(self, cascade = None):
		if len(cascade) > 0 and 'options' in cascade[0]:
			return True
		
		return False
	
	## 
	#  @brief Brief
	#  
	#  @param [in] self Parameter_Description
	#  @param [in] cascade Parameter_Description
	#  @return instrumentVolume, orderId or None, None
	#  
	#  @details Details
	#  	
	def getReverseParams(self, cascade = None):
		if len(cascade) > 0 and 'options' in cascade[0]:
			#case reverse cascade
			return cascade[0]['options']['amount'], cascade[0]['options']['order_id']
			
		idx = len(cascade) - 1
		if 'orderId' in cascade[idx]['sellOrder'] and cascade[idx]['sellOrder']['status'] == 0:
			#case direct cascade
			return cascade[idx]['sellOrder']['operationAmount'], cascade[idx]['sellOrder']['orderId']
		
		return None, None
	
	def inWork(self, cascade = None):
		if (cascade is None):
			cascade = self.cascade
		if (cascade is None):
			print('inWork. Cascade element not defined')
			quit()
		
		workAction = 'buyOrder'
		profitAction = 'sellOrder'
		
		if self.isRevers(cascade):
			workAction, profitAction = profitAction, workAction
		
		byedStage = None
		for element in cascade:
			if self.__isCompleteOrder(element[workAction]):
				byedStage = element['stage']
		
		if byedStage is None:
			return False
		
		for element in cascade:
			if element['stage'] == byedStage:
				if self.__isCompleteOrder(element[profitAction]):
					return False
		
		return True
	
	def needRestart(self, cascade = None):
		if (cascade is None):
			cascade = self.cascade
		if (cascade is None):
			print('needRestart. Cascade element not defined')
			quit()

		if not self.spec.loadTickers([self.pair]):
			quit()
		
		lastPrice = self.spec.tickers[self.pair]['last']
		if self.isRevers(cascade):
			cascadeStartPrice = cascade[0]['sellOrder']['price']
			if lastPrice < cascadeStartPrice * (100 - self.startPercent * 2 ) / 100:
				return True
		else:
			cascadeStartPrice = cascade[0]['buyOrder']['price']
			
			if lastPrice > cascadeStartPrice * (100 + self.startPercent * 2 ) / 100:
				return True
		
		return False
		
	def createOrders(self, cascade = None):
		if (cascade is None):
			cascade = self.cascade
		if (cascade is None):
			print('createOrders. Cascade element not defined')
			quit()
		
		workAction = 'buyOrder'
		profitAction = 'sellOrder'
		
		if self.isRevers(cascade):
			workAction, profitAction = profitAction, workAction
		
		byedStage = None
		createdOrderCount = 0
		for element in cascade:
			if self.__isActiveOrder(element[workAction]):
				createdOrderCount += 1
			if createdOrderCount < self.activeOrdersCount and not self.__isCreatedOrder(element[workAction]):
				orderId = self.spec.createOrder(element[workAction])
				if orderId is False:
					print(self.spec.getLastErrorMessage())
					break
				element[workAction]['orderId'] = orderId
				if orderId is None:
					element[workAction]['status'] = 1
				else:
					element[workAction]['status'] = 0
				createdOrderCount += 1
			
			if self.__isCompleteOrder(element[workAction]):
				byedStage = element['stage']
		
		soldAmount = None
		for element in cascade:
			if element['stage'] > byedStage:
				break
			
			if not soldAmount is None:
				element[profitAction]['operationAmount'] -= soldAmount
			
			if element['stage'] < byedStage and self.__isCompleteOrder(element[profitAction]):
				print('Partial execution in stage {0}'.format(element['stage']))
				element[profitAction]['status'] = 2
				soldAmount = round(element[profitAction]['operationAmount'], self.totalPrecision)
			
			if element['stage'] < byedStage and self.__isActiveOrder(element[profitAction]):
				res = self.spec.cancelOrder(element[profitAction]['orderId'])
				if not res:
					print(self.spec.getLastErrorMessage())
					break
				element[profitAction]['status'] = 2
			
			if element['stage'] == byedStage and not self.__isCreatedOrder(element[profitAction]):
				orderId = self.spec.createOrder(element[profitAction])
				if orderId is False:
					print(self.spec.getLastErrorMessage())
					break
				element[profitAction]['orderId'] = orderId
				if orderId is None:
					element[profitAction]['status'] = 1
				else:
					element[profitAction]['status'] = 0
			
		# fix rest profit orders in partial execution case
		if not soldAmount is None:
			for element in cascade:
				if element['stage'] > byedStage:
					element[profitAction]['operationAmount'] -= soldAmount
		
		return cascade
	
	def cancelOrders(self, cascade = None):
		if (cascade is None):
			cascade = self.cascade
		if (cascade is None):
			print('cancelOrders. Cascade element not defined')
			quit()
		
		workAction = 'buyOrder'
		profitAction = 'sellOrder'
		
		if self.isRevers(cascade):
			workAction, profitAction = profitAction, workAction

		for element in cascade:
			if self.__isActiveOrder(element[workAction]):
				res = self.spec.cancelOrder(element[workAction]['orderId'])
				if not res:
					print(self.spec.getLastErrorMessage())
					break
				element[workAction]['status'] = 2
			
			if self.__isActiveOrder(element[profitAction]):
				print('CANCEL profit order!')
				res = self.spec.cancelOrder(element[profitAction]['orderId'])
				if not res:
					print(self.spec.getLastErrorMessage())
					break
				element[profitAction]['status'] = 2
		
		return cascade
	
	def checkOrders(self, cascade = None):
		if (cascade is None):
			cascade = self.cascade
		if (cascade is None):
			print('checkOrders. Cascade element not defined')
			quit()
		
		for element in cascade:
			if self.__isActiveOrder(element['buyOrder']):
				status = self.spec.getOrderStatus(element['buyOrder']['orderId'])
				if status is False:
					print(self.spec.getLastErrorMessage())
					break
				element['buyOrder']['status'] = status
			
			if self.__isActiveOrder(element['sellOrder']):
				status = self.spec.getOrderStatus(element['sellOrder']['orderId'])
				if status is False:
					print(self.spec.getLastErrorMessage())
					break
				element['sellOrder']['status'] = status
		
		return cascade
	
	def getProfit(self, cascade = None):
		if (cascade is None):
			cascade = self.cascade
		if (cascade is None):
			print('getProfit. Cascade element not defined')
			quit()
		
		profit = False
		
		if self.isRevers(cascade):
			accepted = 0
			for element in cascade:
				accepted += round(element['sellOrder']['price'] * element['sellOrder']['operationAmount'] * (100 - self.fee) / 100, self.pricePrecision)
				if self.__isCompleteOrder(element['buyOrder']):
					used = round(element['buyOrder']['price'] * element['buyOrder']['operationAmount'], self.totalPrecision)
					profit = accepted - used
					print('Stage: {0}, profit: {1}'.format(element['stage'], profit))
					profit = round(profit, self.profitPrecision)
		else:
			invested = 0
			for element in cascade:
				invested += round(element['buyOrder']['price'] * element['buyOrder']['operationAmount'], self.totalPrecision)
				if self.__isCompleteOrder(element['sellOrder']):
					accepted = round(element['sellOrder']['price'] * element['sellOrder']['operationAmount'] * (100 - self.fee) / 100, self.totalPrecision)
					profit = accepted - invested
					print('Stage: {0}, profit: {1}'.format(element['stage'], profit))
					profit = round(profit, self.profitPrecision)
		
		return profit
	
	## 
	#  @brief Brief
	#  
	#  @param [in] self Parameter_Description
	#  @param [in] cascade Parameter_Description
	#  @param [in] cascadeFileName Parameter_Description
	#  @return true in success case
	#  
	#  @details cancel last sell order and save direct cascade
	#  	
	def saveCascade(self, cascade = None, cascadeFileName = None):
		if (cascade is None):
			cascade = self.cascade
		if (cascade is None):
			print('saveCascade. Cascade element not defined')
			return False
		if cascadeFileName is None:
			print('saveCascade. Cascade file name not defined')
			return False
		
		idx = len(cascade) - 1
		
		if not self.__isActiveOrder(cascade[idx]['sellOrder']):
			print('saveCascade. Last save order is not active')
			return False
		
		res = self.spec.cancelOrder(cascade[idx]['sellOrder']['orderId'])
		if not res:
			print('saveCascade. Cancel order error ' + self.spec.getLastErrorMessage())
			return False

		os.rename(cascadeFileName, cascadeFileName + '.bkp')
		return True
	
	## 
	#  @brief Brief
	#  
	#  @param [in] self Parameter_Description
	#  @param [in] cascade Parameter_Description
	#  @param [in] cascadeFileName Parameter_Description
	#  @return Return_Description
	#  
	#  @details Details
	#  	
	def restoreCascade(self, cascade = None, cascadeFileName = None):
		if (cascade is None):
			cascade = self.cascade
		if (cascade is None):
			print('saveCascade. Cascade element is not defined')
			return False
		if cascadeFileName is None:
			print('saveCascade. Cascade file name is not defined')
			return False
	
		os.rename(cascadeFileName + '.bkp', cascadeFileName)
		
		file = open(cascadeFileName, 'r+')
		cascade = json.load(file)
		file.close()
		
		idx = len(cascade) - 1
		
		orderId = self.spec.createOrder(cascade[idx]['sellOrder'])
		if orderId is False:
			print('restoreCascade. Create order error ' + self.spec.getLastErrorMessage())
			return False
			
		cascade[idx]['sellOrder']['orderId'] = orderId
		if orderId is None:
			cascade[idx]['sellOrder']['status'] = 1
		else:
			cascade[idx]['sellOrder']['status'] = 0
		
		file = open(cascadeFileName, 'w+')
		file.write(json.dumps(cascade))
		file.close()
			
		return True

	def __isActiveOrder(self, order):
		if 'orderId' in order and 'status' in order and order['status'] == 0:
			return True
		return False
		
	def __isCreatedOrder(self, order):
		if 'orderId' in order:
			return True
		return False
		
	def __isCompleteOrder(self, order):
		if 'orderId' in order and 'status' in order and order['status'] == 1:
			return True
		return False
Beispiel #18
0
	def __init__(self, key = None, secret = None, silent = False):
		self.spec = Spec(key, secret, silent)
Beispiel #19
0
def ATCS_APD(jobs: list,
             machines: list,
             setup_times: list,
             tao: float = 0.5,
             R: float = 0.8):
    """ ATCS_APD mette insieme l'ATCS index e l'APD, dato che APD tiene conto
		in maniera più precisa dei setup times

	Args:
		jobs (list): [description]
		machines (list): [description]
		setup_times (list): [description]
		tao (float, optional): [description]. Defaults to 0.5.
		R (float, optional): [description]. Defaults to 0.8.
	"""

    # calcolo s come media dei setup times
    # calcolo p come media dei processing times
    s = Spec.get_setup_time_avg(setup_times)
    p = Job.get_processing_time_avg(jobs)

    # calcolo parametri di ATCS
    K1 = 1.2 * np.log(len(jobs) / len(machines)) - R
    #if tao<0.5 or (nano<0.5 and micro>0.5):
    if tao < 0.5:
        K1 -= 0.5
    A2 = 1.8 if tao < 0.8 else 2.0
    K2 = tao / (A2 * math.sqrt(s / p))
    wi = 1

    U = deepcopy(jobs)

    print_schedule_d(machines, 2)

    # step 0 -  Calcolo l'APD per ogni job e setto t=0
    apds = APD(U, setup_times)
    t = 0

    # Step 1 - Prendo la macchina libera al tempo t, poi
    # Per ogni job non schedulato:
    #	a. Calcolo l'ATCS_APD index
    #	b. Assegno alla macchina selezionata il job con ATCS_APD index maggiore
    #	c. Imposto t come il loading machine
    print_d("Index APD_ATCS:", 3)
    while U:
        I_i = dict()
        # prendo la macchina libera al tempo t
        k = Machine.free_machine_at_t(machines, t)
        # job_j è l'ultimo job processato dalla macchina selezionata. Serve per il setup time
        job_j = k._jobs[-1] if k._jobs else None
        # a. calcolo l'ATCS_APD index
        for job_u in U:
            apd_i = apds[job_u]
            I_i[job_u] = ((wi / job_u._processing_time) * math.exp(
                -max(job_u._due_date - job_u._processing_time - t, 0) /
                (K1 * p)) * math.exp(-Job.get_setup_time(job_j, job_u) /
                                     (K2 * s)) * math.exp(-1 / (apd_i * s)))

        I_i = dict(sorted(I_i.items(), key=lambda item: item[1], reverse=True))

        # b. assegno alla macchina il job con index maggiore alla macchina
        for j in I_i:
            # c. imposto il tempo t al loading time
            t = k.get_free_time() + Job.get_setup_time(job_j, j)
            print_d(j, 3)
            k._jobs.append(j)
            U.remove(j)
            break

    print_d("Scheduling APD_ATCS:", 3)
    print_schedule_d(machines, 3)
Beispiel #20
0
def ATCS(jobs, machines, setup_times, tao=0.9, R=0.2):
    """ Calcola l'ATCS index, che tiene conto dei setup times, oltre che
		il processing time ed il due date

	Args:
		jobs (list): [description]
		machines (list): [description]
		setup_times (list): [description]
		tao (float, optional): [description]. Defaults to 0.5.
		R (float, optional): [description]. Defaults to 0.8.
	"""
    # calcolo s come media dei setup times
    # calcolo p come media dei processing times
    s = Spec.get_setup_time_avg(setup_times)
    p = Job.get_processing_time_avg(jobs)

    # calcolo parametri di ATCS
    K1 = 1.2 * np.log(len(jobs) / len(machines)) - R
    #if tao<0.5 or (nano<0.5 and micro>0.5):
    if tao < 0.5:
        K1 -= 0.5
    A2 = 1.8 if tao < 0.8 else 2.0
    K2 = tao / (A2 * math.sqrt(s / p))
    wi = 1

    U = deepcopy(jobs)

    print_schedule_d(machines, 2)

    # Step 1 - Prendo la macchina libera al tempo t, poi
    # Per ogni job non schedulato:
    #	a. Calcolo l'ATCS index
    #	b. Assegno alla macchina selezionata il job con ATCS_APD index maggiore
    print_d("Index ATCS:", 2)
    while U:
        I_i = dict()
        # prendo la macchina libera al tempo t
        k = Machine.first_machine_avaible(machines)
        # job_j è l'ultimo job processato dalla macchina selezionata. Serve per il setup time
        job_j = k._jobs[-1] if k._jobs else None
        # a. calcolo l'ATCS index
        for job_u in U:
            tmp_machine = deepcopy(k)
            tmp_machine._jobs.append(job_u)
            t = tmp_machine.get_free_time()
            I_i[job_u] = ((wi / job_u._processing_time) * math.exp(
                -max(job_u._due_date - job_u._processing_time - t, 0) /
                (K1 * p)) * math.exp(-Job.get_setup_time(job_j, job_u) /
                                     (K2 * s)))

        I_i = dict(sorted(I_i.items(), key=lambda item: item[1], reverse=True))

        # b. assegno alla macchina il job con index maggiore alla macchina
        for j in I_i:
            print_d(j, 2)
            k._jobs.append(j)
            U.remove(j)
            break

    print_d("Scheduling ATCS:", 2)
    print_schedule_d(machines, 2)
Beispiel #21
0
"""
	
import config
from spec import Spec
import os.path

key = None
secret = None

if os.path.isfile('../arbitrage.key'):
	f = open('../arbitrage.key', 'r')
	key = f.readline().strip()
	secret = f.readline().strip()
	f.close()

curSite = Spec(key, secret)

if not curSite.checkConnection():
	quit()

if not curSite.loadTradeConditions():
	quit()
	
if not curSite.generateTradeSequence(config.startCurrency, config.tradeSequence, config.tradeLength):
	quit()

if curSite.hasSavedTrades():
	selectedTrades = curSite.loadTrades()
	
	key = 0
	cont = True
Beispiel #22
0
class Scriptor:
    """
    The phases of processing:
    - Set up data structures and global attributes
    - Prepare image specs and organize them
    - Launch one thread to initialize image spec (read input image, set up (random) attributes for animation and transition)
    - Launch multiple threads to generate frames and store results
    - In main thread: wait for results and write to video
    - Join video with audio
    """
    def generateVideo(self, scriptFile):
        with open(scriptFile) as t:
            self.rootSpec = Spec(yaml.safe_load(t), None)

        rootSpec = self.rootSpec
        self.framerate = rootSpec.get('framerate', 30)
        self.frameWidth = rootSpec.get('framewidth', 1440)
        self.frameHeight = rootSpec.get('frameheight', 1080)
        self.outputFrames = rootSpec.get('outputframes')
        self.limitFrames = rootSpec.get('limitframes')
        random.seed(rootSpec.get('randomseed'))

        outputFile = rootSpec.get('outputfile', 'video.mp4')
        videoOut = outputFile + '.temp.mp4'

        # Initialize data structures
        self.imageSpecQueue = queue.Queue()
        self.imageFrameQueue = queue.Queue()
        self.resultQueue = queue.Queue()
        self.prevSpec = None
        self.allImageSpecsInitialized = False

        # Prepare data structures for processing
        images = rootSpec.get('images', [])
        self.prepareImageSpecs(images, rootSpec)

        # Start one thread to initialize image specs
        threading.Thread(target=self.runnableInitImageSpecs).start()
        
        # Start processing image specs by launching worker threads
        self.globalFrameN = 0
        for _ in range(self.rootSpec.get('threads', 16)):
            threading.Thread(target=self.runnableProcessFrame).start()

        # In the current thread, wait for and write the results
        self.writer = imageio.get_writer(videoOut, 
            fps=self.framerate,
            macro_block_size=8)
        self.processResults()
        self.writer.close()

        # Join audio
        audioSpec = rootSpec.getSpec('audio')
        if not audioSpec is None:
            self.combineVideoWithAudio(audioSpec, videoOut, outputFile)
    
    def prepareImageSpecs(self, images, parentSpec):
        """ Walks through the image specs recursively, in order, links them, adds them to the queues.
        """
        for item in images:
            itemSpec = ImageSpec(item, parentSpec)

            subgroup = itemSpec.get('images', None, doRecurse=False)
            if subgroup is None:
                # Set required variable in prev spec from current spec
                if not self.prevSpec is None:
                    self.prevSpec.nextTransitionDuration = itemSpec.get('transitiontime', 0)

                # Link and remember previous
                itemSpec.prevSpec = self.prevSpec
                self.prevSpec = itemSpec

                # Put in queues
                self.imageSpecQueue.put(itemSpec)
                self.resultQueue.put(itemSpec)
            else:
                # Recurse
                self.prepareImageSpecs(subgroup, itemSpec)
    
    def runnableInitImageSpecs(self):
        # Initialize image specs while they are available
        # (the queue is pre-filled, so when it's empty, we're done)
        imageSpec = getFromQueue(self.imageSpecQueue)
        while not imageSpec is None:
            self.initializeImageSpec(imageSpec)

            # Wait with initializing next image spec.
            # (we don't want to initialize and load too early, to limit memory usage,
            # but we also don't want to load too late, because it will block the threads,
            # so start loading when we have less than a certain amount of frames to process)
            while self.imageFrameQueue.qsize() > 60:
                time.sleep(0.1)

            imageSpec = getFromQueue(self.imageSpecQueue)
        
        # Flag that allows frame processing threads to finish if there are no more frames
        self.allImageSpecsInitialized = True
        print("finished processing image specs")

    def initializeImageSpec(self, imageSpec):
        # Read image
        inputFileName = imageSpec.get('file')
        #assert not inputFileName is None, 'No input file specified'
        if inputFileName is None:
            npImCurrent = np.zeros((self.frameHeight, self.frameWidth, 3), dtype='uint8')
        else:
            npImCurrent = imageio.imread('./input/%s' % inputFileName)
        
        # Set up transition
        #TODO: relfect: transitionType = transitionSpec.get('type', 'blend')
        imageSpec.transition = BlendTransition()

        # Set up animation
        #animationType = animationSpec.get(Props.IMAGE_ANIMATION_TYPE)
        #TODO: use reflection to instantiate:
        imageSpec.animation = PanZoomAnimation(npImCurrent, imageSpec)
        imageSpec.duration = duration = imageSpec.get('duration', 2.0)
        

        nframes = int(duration * self.framerate)
        imageSpec.frames = [None] * nframes
        for i in range(0, nframes):
            self.imageFrameQueue.put((imageSpec, i))

    def runnableProcessFrame(self):
        imageFrame = getFromQueue(self.imageFrameQueue)
        while (not imageFrame is None) or (not self.allImageSpecsInitialized):

            # Either we have an image to process or we have to wait for one
            if not imageFrame is None:
                (imageSpec, frameNr) = imageFrame
                self.processFrame(imageSpec, frameNr)
            else:
                time.sleep(0.1)

            imageFrame = getFromQueue(self.imageFrameQueue)

        print("finished processing frames")
        
    def processFrame(self, imageSpec, i):
        prevSpec = imageSpec.prevSpec
        transitionDuration = imageSpec.get('transitiontime', 0.5)
        duration = imageSpec.duration
        nextTransitionDuration = imageSpec.nextTransitionDuration
        animation = imageSpec.animation
        transition = imageSpec.transition

        print("processing %s frame %d/%d" % (imageSpec.get('file'), i + 1, len(imageSpec.frames)))

        # Animate image
        animationT = self.getTForFrame(i, duration + nextTransitionDuration, self.framerate)
        npIm1 = animation.animate(animationT)
        
        # Transition
        transitionT = 1.0 if (transitionDuration == 0) else i / (transitionDuration * self.framerate)
        if transitionT < 1.0 and not prevSpec is None:
            # Animate previous image
            animationT = self.getTForFrame(prevSpec.duration * self.framerate + i,
                prevSpec.duration + transitionDuration, self.framerate)
            npIm0 = prevSpec.animation.animate(animationT)

            # Combine transition images
            npResult = transition.processTransition(npIm0, npIm1, transitionT)
        else:
            npResult = npIm1

        # Put result in list to be written
        imageSpec.frames[i] = npResult

    def processResults(self):
        # self.resultQueue has been prefilled (to keep results in the correct order),
        # so we just need to keep going until it is empty
        currentResult = getFromQueue(self.resultQueue)
        while not currentResult is None:

            # Wait for initialization
            while currentResult.frames is None:
                time.sleep(0.1)

            # Wait for and process each result frame in order
            for i in range(len(currentResult.frames)):
                # Wait for result
                while currentResult.frames[i] is None:
                    time.sleep(0.1)
                # Process result
                self.writeResultImage(currentResult.frames[i])
                # Clean up
                currentResult.frames[i] = None

            # Clean up unused references to free memory
            if not currentResult.prevSpec is None:
                # Clean up finished spec so memory can be released
                currentResult.prevSpec.animation = None
                currentResult.prevSpec.transition = None
                currentResult.prevSpec = None

            currentResult = getFromQueue(self.resultQueue)

    def writeResultImage(self, image):
        # Write frame to video
        self.writer.append_data(image)

        # Write frame to image if set up
        if not self.outputFrames is None:
            imageio.imwrite(self.outputFrames % self.globalFrameN, image)
        self.globalFrameN += 1

    # Scales the frameNumber to the current position in the animation to a fraction
    # between 0 and 1.
    # totalDuration should include the animation duration for the current image and the
    # transition duration to the next image
    def getTForFrame(self, frameNumber, totalDuration, frameRate):
        return frameNumber / (totalDuration * frameRate)
    
    def combineVideoWithAudio(self, audioSpec, videoIn, videoOut):
        def maybe(option, key, spec):
            value = spec.get(key)
            return [option, str(value)] if not value is None else []
        audioIn = audioSpec.get('file')
        cmd_out = ['ffmpeg',
                '-y',
                '-i', videoIn,
                *maybe('-ss', 'audiooffset', audioSpec),
                *maybe('-itsoffset', 'videooffset', audioSpec),
                '-i', audioIn,
                #'-c', 'copy',
                '-map', '0:v',
                '-map', '1:a',
                '-shortest',
                '-filter:a', 'afade=t=out:st=144:d=8',
                videoOut]
        pipe = subprocess.Popen(cmd_out)
        pipe.wait()