def train(args):
check_present(
args,
["train_corpus", "output_folder", "dev_corpus", "train_shuffle_seed"])
train_corpus_path = args.train_corpus
if args.train_shuffle_seed > 0:
reader = sling.RecordReader(args.train_corpus)
items = [(key, value) for key, value in reader]
reader.close()
r = random.Random(args.train_shuffle_seed)
r.shuffle(items)
train_corpus_path = os.path.join(args.output_folder,
"train_shuffled.rec")
writer = sling.RecordWriter(train_corpus_path)
for key, value in items:
writer.write(key, value)
writer.close()
print("Wrote shuffled train corpus to %s using seed %d" % \
(train_corpus_path, args.train_shuffle_seed))
# Setting an explicit seed for the sake of determinism.
torch.manual_seed(1)
# Make commons store if needed.
if args.commons == '' or not os.path.exists(args.commons):
if args.commons == '':
fname = os.path.join(args.output_folder, "commons")
print("Will create a commons store at", fname)
args.commons = fname
else:
print("No commons found at", args.commons, ", creating it...")
_, symbols = commons_builder.build(
[train_corpus_path, args.dev_corpus], args.commons)
print("Commons created at", args.commons, "with", len(symbols), \
"symbols besides the usual ones.")
# Make the training spec.
spec = Spec()
spec.build(args.commons, train_corpus_path)
# Initialize the model with the spec and any word embeddings.
caspar = Caspar(spec)
embeddings_file = args.word_embeddings
if embeddings_file == '': embeddings_file = None
caspar.initialize(embeddings_file)
tmp_folder = os.path.join(args.output_folder, "tmp")
if not os.path.exists(tmp_folder):
os.makedirs(tmp_folder)
evaluator = partial(dev_accuracy, args.dev_corpus, tmp_folder)
output_file_prefix = os.path.join(args.output_folder, "caspar")
hyperparams = Hyperparams(args)
print("Using hyperparameters:", hyperparams)
trainer = Trainer(caspar, hyperparams, evaluator, output_file_prefix)
train = Corpora(train_corpus_path, spec.commons, gold=True)
trainer.train(train)
def spec_i(self, i_mock):
'''
k, power/bispectrum (p0k, p2k, p4k, bk) of ith mock
'''
specdict = self.cat_corr['spec']
# copy cat_corr dictionary
cat_corr_i = self.cat_corr.copy()
cat_corr_i['catalog']['n_mock'] = i_mock
spec_i = Spec(self.type, cat_corr_i, **self.kwargs)
spec_i.read()
if self.type == 'pk':
spec_ell = ''.join(['p', str(specdict['ell']), 'k'])
spec_i_spec = getattr(spec_i, spec_ell)
return [spec_i.k, spec_i_spec]
elif self.type == 'bk':
bk_i = getattr(spec_i, 'bk')
qk_i = getattr(spec_i, 'qk')
return [spec_i.k1, spec_i.k2, spec_i.k3, bk_i, qk_i]
else:
raise NotImplementedError
def build_bk_wrapper(params):
""" Wrapper for calculating power/bispectrum
"""
cat_corr = params[0]
kwargs = {}
if len(params) > 2:
kwargs = params[1]
spectrum = Spec('bk', cat_corr, **kwargs)
print spectrum.file()
spectrum.build()
return None
class _SpecPluginTestCase(PluginTester, unittest.TestCase):
activate = '--with-spec'
args = ['--no-spec-color']
plugins = [Spec()]
def _get_suitepath(self):
return '_spec_test_cases/%s.py' % self.suitename
suitepath = property(_get_suitepath)
def assertContains(self, needle, haystack):
assert needle in haystack,\
"Failed to find:\n\n%s\ninside\n%s\n" % \
(_prepend_in_each_line(needle), _prepend_in_each_line(haystack))
def assertContainsInOutput(self, string):
self.assertContains(string, six.text_type(self.output))
def failIfContains(self, needle, haystack):
assert needle not in haystack,\
"Found:\n\n%s\ninside\n%s\n" % \
(_prepend_in_each_line(needle), _prepend_in_each_line(haystack))
def failIfContainsInOutput(self, string):
self.failIfContains(string, six.text_type(self.output))
class TestPluginSpecWithFoobazAndStandardPluginsEnabled(
TestPluginSpecWithFoobaz):
plugins = [
Spec(),
nose.plugins.skip.Skip(),
nose.plugins.deprecated.Deprecated()
]
def __init__(self, *args, **kwargs):
super(TopoSpec, self).__init__(*args, **kwargs)
app_manager.require_app("ryu.topology.switches")
CONF.register_opt(StrOpt("spec_path", default="spec/mininet.yml"))
self.graph = nx.Graph()
self.spec = Spec.from_yaml(CONF.spec_path)
self._thread = hub.spawn_after(3, self._compare_spec)
def main(self):
""" Main function.
Entry point of the program.
"""
try:
from spec import Spec
self.setup_parser()
args = self.parser.parse_args()
self.log = get_logger()
if args.verbose:
self.log.setLevel('INFO')
if args.debug:
self.log.setLevel('DEBUG')
pypi = Pypi2spec(args.package)
pypi.retrieve_info()
pypi.download()
pypi.extract_sources()
pypi.determine_arch()
pypi.remove_sources()
settings = Settings()
spec = Spec(settings, pypi)
spec.fill_spec_info()
spec.get_template()
spec.write_spec()
except Pypi2specError, err:
print err
def train(args):
check_present(args, ["train_corpus", "output_folder", "dev_corpus"])
# Setting an explicit seed for the sake of determinism.
torch.manual_seed(1)
# Make commons store if needed.
if args.commons == '' or not os.path.exists(args.commons):
if args.commons == '':
fname = os.path.join(args.output_folder, "commons")
print "Will create a commons store at", fname
args.commons = fname
else:
print "No commons found at", args.commons, ", creating it..."
_, symbols = commons_builder.build(
[args.train_corpus, args.dev_corpus], args.commons)
print "Commons created at", args.commons, "with", len(symbols), \
"symbols besides the usual ones."
# Make the training spec.
spec = Spec()
spec.build(args.commons, args.train_corpus)
# Initialize the model with the spec and any word embeddings.
caspar = Caspar(spec)
embeddings_file = args.word_embeddings
if embeddings_file == '': embeddings_file = None
caspar.initialize(embeddings_file)
tmp_folder = os.path.join(args.output_folder, "tmp")
if not os.path.exists(tmp_folder):
os.makedirs(tmp_folder)
evaluator = partial(dev_accuracy,
args.commons,
args.dev_corpus,
tmp_folder)
output_file_prefix = os.path.join(args.output_folder, "caspar")
hyperparams = Hyperparams(args)
print "Using hyperparameters:", hyperparams
trainer = Trainer(caspar, hyperparams, evaluator, output_file_prefix)
train = Corpora(args.train_corpus, spec.commons, gold=True)
trainer.train(train)
def generateVideo(self, scriptFile):
with open(scriptFile) as t:
self.rootSpec = Spec(yaml.safe_load(t), None)
rootSpec = self.rootSpec
self.framerate = rootSpec.get('framerate', 30)
self.frameWidth = rootSpec.get('framewidth', 1440)
self.frameHeight = rootSpec.get('frameheight', 1080)
self.outputFrames = rootSpec.get('outputframes')
self.limitFrames = rootSpec.get('limitframes')
random.seed(rootSpec.get('randomseed'))
outputFile = rootSpec.get('outputfile', 'video.mp4')
videoOut = outputFile + '.temp.mp4'
# Initialize data structures
self.imageSpecQueue = queue.Queue()
self.imageFrameQueue = queue.Queue()
self.resultQueue = queue.Queue()
self.prevSpec = None
self.allImageSpecsInitialized = False
# Prepare data structures for processing
images = rootSpec.get('images', [])
self.prepareImageSpecs(images, rootSpec)
# Start one thread to initialize image specs
threading.Thread(target=self.runnableInitImageSpecs).start()
# Start processing image specs by launching worker threads
self.globalFrameN = 0
for _ in range(self.rootSpec.get('threads', 16)):
threading.Thread(target=self.runnableProcessFrame).start()
# In the current thread, wait for and write the results
self.writer = imageio.get_writer(videoOut,
fps=self.framerate,
macro_block_size=8)
self.processResults()
self.writer.close()
# Join audio
audioSpec = rootSpec.getSpec('audio')
if not audioSpec is None:
self.combineVideoWithAudio(audioSpec, videoOut, outputFile)
def run(args):
check_present(args, ["input", "parser", "output"])
assert os.path.exists(args.input), args.input
assert os.path.exists(args.parser), args.parser
# Read parser flow.
flow = Flow()
flow.load(args.parser)
# Initialize the spec from the flow.
spec = Spec()
spec.from_flow(flow)
# Initialize the model from the flow.
caspar = Caspar(spec)
caspar.from_flow(flow)
corpus = Corpora(args.input, caspar.spec.commons)
writer = sling.RecordWriter(args.output)
count = 0
for document in corpus:
state, _, _, trace = caspar.forward(document,
train=False,
debug=args.trace)
state.write()
if trace:
trace.write()
writer.write(str(count), state.encoded())
count += 1
if count % 100 == 0:
print "Annotated", count, "documents", now(), mem()
writer.close()
print "Annotated", count, "documents", now(), mem()
print "Wrote annotated documents to", args.output
if args.evaluate:
f = tempfile.NamedTemporaryFile(delete=False)
fname = f.name
caspar.spec.commons.save(fname, binary=True)
f.close()
eval_result = frame_evaluation(gold_corpus_path=args.input, \
test_corpus_path=args.output, commons=caspar.spec.commons)
os.unlink(fname)
return eval_result
def set_components(top):
spec = Spec.get_spec('Iso8583 v0 - Test')
top.title = top.title + ' - ' + spec.name
spec_tree = ttk.Treeview(top, columns=('Field', 'Field Encoding', 'Field Value'), selectmode='browse')
msg = spec.get_msg_by_name('Authorization Message - 0100')
for field_def in msg.req_fields:
print field_def.name
spec_tree.insert('', 'end', field_def.name, text=field_def.name,
values=field_def.name + ',' + field_def.encoding)
spec_tree.pack()
class TestPluginSpecWithDoctestsButDisabled(_SpecPluginTestCase):
activate = '--with-spec'
# no --spec-doctests option
args = ['--with-doctest', '--doctest-tests', '--no-spec-color']
plugins = [Spec(), nose.plugins.doctests.Doctest()]
suitename = 'doctests'
def test_doesnt_build_specifications_for_doctests_when_spec_doctests_option_wasnt_set(
self):
self.failIfContainsInOutput("test_doctests")
self.failIfContainsInOutput("2 + 3 returns 5")
class TestPluginSpecWithDoctests(_SpecPluginTestCase):
activate = '--with-spec'
args = [
'--with-doctest', '--doctest-tests', '--spec-doctests',
'--no-spec-color'
]
plugins = [Spec(), nose.plugins.doctests.Doctest()]
suitename = 'doctests'
expected_test_doctests_output = """doctests
- 2 + 3 returns 5
- None is nothing
- foobar throws "NameError: name 'foobar' is not defined"
"""
def test_builds_specifications_for_doctests(self):
self.assertContainsInOutput(self.expected_test_doctests_output)
machines = [Machine(1), Machine(2)]
# setup times dalla tabella 2 dell'articolo
setup_times = {
"length": 60,
"width": 15,
"thickness": 10,
"hardness": 20,
"colour": 15,
}
# creazione delle specs per ogni job, dalla tabelle 2 dell'articolo
specs_list = {
1: [
Spec("length", 240, setup_times["length"]),
Spec("width", 12, setup_times["width"]),
Spec("thickness", 1.5, setup_times["thickness"]),
Spec("hardness", 7, setup_times["hardness"]),
Spec("colour", 1, setup_times["colour"])
],
2: [
Spec("length", 96, setup_times["length"]),
Spec("width", 36, setup_times["width"]),
Spec("thickness", 5.0, setup_times["thickness"]),
Spec("hardness", 8, setup_times["hardness"]),
Spec("colour", 1, setup_times["colour"])
],
3: [
Spec("length", 96, setup_times["length"]),
Spec("width", 24, setup_times["width"]),
def set_components(top):
spec = Spec.get_spec('Iso8583 v0 - Test')
top.title = top.title + ' - ' + spec.name
spec_tree = ttk.Treeview(top, columns=('Field', 'Field Encoding', 'Field Value'), selectmode='browse')
msg = spec.get_msg_by_name('Authorization Message - 0100')
for field_def in msg.req_fields:
print field_def.name
spec_tree.insert('', 'end', field_def.name, text=field_def.name,
values=field_def.name + ',' + field_def.encoding)
spec_tree.pack()
if __name__ == "__main__":
Spec.register_spec(Iso8583v0Spec())
top = Tk.Tk()
top.title = 'ISO8583 Simulator'
top.configure(height=600, width=600)
set_components(top)
top.mainloop()
# Spec.register_spec(Iso8583v0Spec())
# spec = Spec.get_spec('Iso8583 v0 - Test')
# msg = spec.parse(binascii.unhexlify(
# 'f1f1f0f0' + 'F3000000200000018000000000000001' + '0F313231313131313131313131313134' + '303034303030' +
# '31323334353637383930' + '30333033' + '313130303030' + 'F0F1F5F1F2F1F1F1F1F1F1F1F1F1F1F1F1F4' +
# 'e3456af57b0198f4' + 'F0F0F4F0F0F0F8' + 'e3456af57b0198f4'))
def __init__(self):
self.easy = int(input("Enter the marks for easy questions > "))
self.medium = int(input("Enter the marks for medium questions > "))
self.hard = int(input("Enter the marks for hard questions > "))
self.entry = Spec(self.easy, self.medium, self.hard)
self.store = QuestionStore()
class Cascade:
checkTimeout = 10
spec = None
cascade = None
lastPrice = None
pricePrecision = None
minAmount = None
fee = None
activeOrdersCount = 5
totalPrecision = 8
profitPrecision = 2
pair = 'btc_usd'
profitPercent = 1
startPercent = 1
deepPercent = 15
totalInvest = 5
allowRevers = False
def setActiveOrdersCount(self, activeOrdersCount):
self.activeOrdersCount = activeOrdersCount
def setTotalPrecision(self, totalPrecision):
self.totalPrecision = totalPrecision
def setProfitPrecision(self, profitPrecision):
self.profitPrecision = profitPrecision
def setPair(self, pair):
self.pair = pair
if not self.spec.checkConnection():
quit()
if not self.spec.loadTradeConditions():
quit()
if not self.spec.loadTickers([pair]):
quit()
self.lastPrice = self.spec.tickers[pair]['last']
self.pricePrecision = self.spec.pairs[pair]['decimal_places']
self.minAmount = self.spec.pairs[pair]['min_amount']
self.fee = self.spec.pairs[pair]['fee']
def setProfitPercent(self, profitPercent):
self.profitPercent = profitPercent
def setStartPercent(self, startPercent):
self.startPercent = startPercent
def setDeepPercent(self, deepPercent):
self.deepPercent = deepPercent
def setTotalInvest(self, totalInvest):
self.totalInvest = totalInvest
def setAllowRevers(self, allowRevers):
self.allowRevers = allowRevers
def __init__(self, key = None, secret = None, silent = False):
self.spec = Spec(key, secret, silent)
def setCascade(self, cascade):
self.cascade = cascade
def getCascade(self):
return self.cascade
def printCascade(self, cascade = None):
if (cascade is None):
print('printCascade. Cascade element not defined')
quit()
if len(cascade) > 0 and 'options' in cascade[0]:
returned = 0
accepted = 0
profit = 0
for item in cascade:
accepted += round(item['sellOrder']['price'] * item['sellOrder']['operationAmount'] * (100 - self.fee) / 100, self.pricePrecision)
returned = round(item['buyOrder']['operationAmount'] * (100 - self.fee) / 100, self.totalPrecision)
profit = round(accepted - item['buyOrder']['operationAmount'] * item['buyOrder']['price'], self.profitPrecision)
print('{0[stage]:>3} sell {1[operationAmount]:<12}@ {1[price]:<12}acc: {3:<12} buy: {2[operationAmount]:<12}@ {2[price]:<12}ret: {4:<14} {5:<2}'.format(item, item['sellOrder'], item['buyOrder'], accepted, returned, profit))
return
invested = 0
for item in cascade:
invested += round(item['buyOrder']['price'] * item['buyOrder']['operationAmount'], self.totalPrecision)
accepted = round(item['sellOrder']['price'] * item['sellOrder']['operationAmount'] * (100 - self.fee) / 100, self.totalPrecision)
profit = round(accepted - invested, self.profitPrecision)
print('{0[stage]:>3} buy {1[operationAmount]:<12}@ {1[price]:<12}inv: {3:<12} sell: {2[operationAmount]:<12}@ {2[price]:<12}accp: {4:<14} {5:<2}'.format(item, item['buyOrder'], item['sellOrder'], invested, accepted, profit))
def createCascade(self, instrumentVolume = None, orderId = None, maxStages = 150):
self.setPair(self.pair)
if not instrumentVolume is None:
instrumentVolume = float(instrumentVolume)
self.totalInvest = float(self.totalInvest)
if orderId:
startPrice = round(self.lastPrice * (100 + self.startPercent) / 100, self.pricePrecision)
endPrice = round(startPrice * (100 + self.deepPercent) / 100, self.pricePrecision)
priceLength = endPrice - startPrice
investFreq = round(instrumentVolume / priceLength, self.totalPrecision)
investQuant = round(self.minAmount * (100 + 2 * self.fee) / 100, self.totalPrecision)
priceStep = priceLength * investQuant / instrumentVolume
#priceStep = round(priceLength * investQuant / instrumentVolume, self.pricePrecision)
if priceStep < 10 ** -self.pricePrecision:
priceStep = 10 ** -self.pricePrecision
investQuant = round(priceStep * investFreq, self.pricePrecision)
if instrumentVolume // investQuant > maxStages:
investQuant = round(instrumentVolume / maxStages, self.totalPrecision)
priceStep = priceLength / maxStages
#priceStep = round(priceLength / maxStages, self.pricePrecision)
else:
maxStages = int(instrumentVolume // investQuant)
investQuant = round(instrumentVolume / (instrumentVolume // investQuant), self.totalPrecision)
priceStep = priceLength / (instrumentVolume // investQuant)
#priceStep = round(priceLength / (instrumentVolume // investQuant), self.pricePrecision)
acceptedSumm = 0
cascade = []
for stage in range(0, maxStages):
curPrice = round(startPrice + priceStep * stage, self.pricePrecision)
accepted = investQuant * (stage + 1)
acceptedSumm += round(investQuant * curPrice * (100 - self.fee) / 100, self.pricePrecision)
buyAmount = round(accepted * 100 / (100 - self.fee), self.totalPrecision)
buyPrice = round(acceptedSumm / accepted * (100 - self.profitPercent) / 100, self.pricePrecision)
cascade.append({
'stage': stage,
'buyOrder': {'pair': self.pair, 'action': 'buy', 'price': buyPrice, 'operationAmount': buyAmount},
'sellOrder': {'pair': self.pair, 'action': 'sell', 'price': curPrice, 'operationAmount': investQuant},
'options' : {'order_id': orderId, 'amount': instrumentVolume}})
return cascade
startPrice = round(self.lastPrice * (100 - self.startPercent) / 100, self.pricePrecision)
endPrice = round(startPrice * (100 - self.deepPercent) / 100, self.pricePrecision)
priceLength = startPrice - endPrice
investFreq = round((startPrice - endPrice) / self.totalInvest, self.totalPrecision)
investQuant = round(startPrice * self.minAmount * (100 + 2 * self.fee) / 100, self.totalPrecision)
priceStep = investQuant * investFreq
#priceStep = round(investQuant * investFreq, self.pricePrecision)
if investQuant * investFreq < 10 ** -self.pricePrecision:
priceStep = 10 ** -self.pricePrecision
investQuant = round(priceStep / investFreq, self.totalPrecision)
if self.totalInvest // investQuant > maxStages:
investQuant = round(self.totalInvest / maxStages, self.totalPrecision)
priceStep = priceLength / maxStages
#priceStep = round(priceLength / maxStages, self.pricePrecision)
else:
maxStages = int(self.totalInvest // investQuant)
investQuant = round(self.totalInvest / (self.totalInvest // investQuant), self.totalPrecision)
priceStep = priceLength / (self.totalInvest // investQuant)
#priceStep = round(priceLength / (self.totalInvest // investQuant), self.pricePrecision)
sellAmount = 0
cascade = []
for stage in range(0, maxStages):
curPrice = round(startPrice - priceStep * stage, self.pricePrecision)
curAmount = round(investQuant / curPrice, self.totalPrecision)
if curAmount < self.minAmount:
curAmount = self.minAmount
invested = investQuant * (stage + 1)
sellAmount += round(curAmount * (100 - self.fee) / 100, self.totalPrecision)
sellPrice = round(invested / sellAmount * (100 + self.profitPercent) / 100, self.pricePrecision)
cascade.append({'stage': stage, 'buyOrder': {'pair': self.pair, 'action': 'buy', 'price': curPrice, 'operationAmount': curAmount}, 'sellOrder': {'pair': self.pair, 'action': 'sell', 'price': sellPrice, 'operationAmount': sellAmount}})
return cascade
##
# @brief Brief
#
# @param [in] self Parameter_Description
# @param [in] cascade Parameter_Description
# @return True if can start reverse cascade, False in other case
#
# @details 0 - active, 1 - complete, 2 - canceled
#
def needReverse(self, cascade = None):
if (cascade is None):
cascade = self.cascade
if (cascade is None):
return False
if len(cascade) == 0:
return False
if 'options' in cascade[0]:
return False
if not self.allowRevers:
return False
idx = len(cascade) - 1
if 'status' in cascade[idx]['buyOrder'] and 'status' in cascade[idx]['sellOrder']:
if cascade[idx]['buyOrder']['status'] == 1 and cascade[idx]['sellOrder']['status'] == 0:
if round(self.lastPrice * (100 + self.startPercent) / 100, self.pricePrecision) < cascade[idx]['sellOrder']['price']:
return True
return False
##
# @brief detect type of cascade True - reverse, False - Normal
#
# @param [in] self Parameter_Description
# @param [in] cascade Parameter_Description
# @return Return_Description
#
# @details what doing with incorrect cascade structure?
#
def isRevers(self, cascade = None):
if len(cascade) > 0 and 'options' in cascade[0]:
return True
return False
##
# @brief Brief
#
# @param [in] self Parameter_Description
# @param [in] cascade Parameter_Description
# @return instrumentVolume, orderId or None, None
#
# @details Details
#
def getReverseParams(self, cascade = None):
if len(cascade) > 0 and 'options' in cascade[0]:
#case reverse cascade
return cascade[0]['options']['amount'], cascade[0]['options']['order_id']
idx = len(cascade) - 1
if 'orderId' in cascade[idx]['sellOrder'] and cascade[idx]['sellOrder']['status'] == 0:
#case direct cascade
return cascade[idx]['sellOrder']['operationAmount'], cascade[idx]['sellOrder']['orderId']
return None, None
def inWork(self, cascade = None):
if (cascade is None):
cascade = self.cascade
if (cascade is None):
print('inWork. Cascade element not defined')
quit()
workAction = 'buyOrder'
profitAction = 'sellOrder'
if self.isRevers(cascade):
workAction, profitAction = profitAction, workAction
byedStage = None
for element in cascade:
if self.__isCompleteOrder(element[workAction]):
byedStage = element['stage']
if byedStage is None:
return False
for element in cascade:
if element['stage'] == byedStage:
if self.__isCompleteOrder(element[profitAction]):
return False
return True
def needRestart(self, cascade = None):
if (cascade is None):
cascade = self.cascade
if (cascade is None):
print('needRestart. Cascade element not defined')
quit()
if not self.spec.loadTickers([self.pair]):
quit()
lastPrice = self.spec.tickers[self.pair]['last']
if self.isRevers(cascade):
cascadeStartPrice = cascade[0]['sellOrder']['price']
if lastPrice < cascadeStartPrice * (100 - self.startPercent * 2 ) / 100:
return True
else:
cascadeStartPrice = cascade[0]['buyOrder']['price']
if lastPrice > cascadeStartPrice * (100 + self.startPercent * 2 ) / 100:
return True
return False
def createOrders(self, cascade = None):
if (cascade is None):
cascade = self.cascade
if (cascade is None):
print('createOrders. Cascade element not defined')
quit()
workAction = 'buyOrder'
profitAction = 'sellOrder'
if self.isRevers(cascade):
workAction, profitAction = profitAction, workAction
byedStage = None
createdOrderCount = 0
for element in cascade:
if self.__isActiveOrder(element[workAction]):
createdOrderCount += 1
if createdOrderCount < self.activeOrdersCount and not self.__isCreatedOrder(element[workAction]):
orderId = self.spec.createOrder(element[workAction])
if orderId is False:
print(self.spec.getLastErrorMessage())
break
element[workAction]['orderId'] = orderId
if orderId is None:
element[workAction]['status'] = 1
else:
element[workAction]['status'] = 0
createdOrderCount += 1
if self.__isCompleteOrder(element[workAction]):
byedStage = element['stage']
soldAmount = None
for element in cascade:
if element['stage'] > byedStage:
break
if not soldAmount is None:
element[profitAction]['operationAmount'] -= soldAmount
if element['stage'] < byedStage and self.__isCompleteOrder(element[profitAction]):
print('Partial execution in stage {0}'.format(element['stage']))
element[profitAction]['status'] = 2
soldAmount = round(element[profitAction]['operationAmount'], self.totalPrecision)
if element['stage'] < byedStage and self.__isActiveOrder(element[profitAction]):
res = self.spec.cancelOrder(element[profitAction]['orderId'])
if not res:
print(self.spec.getLastErrorMessage())
break
element[profitAction]['status'] = 2
if element['stage'] == byedStage and not self.__isCreatedOrder(element[profitAction]):
orderId = self.spec.createOrder(element[profitAction])
if orderId is False:
print(self.spec.getLastErrorMessage())
break
element[profitAction]['orderId'] = orderId
if orderId is None:
element[profitAction]['status'] = 1
else:
element[profitAction]['status'] = 0
# fix rest profit orders in partial execution case
if not soldAmount is None:
for element in cascade:
if element['stage'] > byedStage:
element[profitAction]['operationAmount'] -= soldAmount
return cascade
def cancelOrders(self, cascade = None):
if (cascade is None):
cascade = self.cascade
if (cascade is None):
print('cancelOrders. Cascade element not defined')
quit()
workAction = 'buyOrder'
profitAction = 'sellOrder'
if self.isRevers(cascade):
workAction, profitAction = profitAction, workAction
for element in cascade:
if self.__isActiveOrder(element[workAction]):
res = self.spec.cancelOrder(element[workAction]['orderId'])
if not res:
print(self.spec.getLastErrorMessage())
break
element[workAction]['status'] = 2
if self.__isActiveOrder(element[profitAction]):
print('CANCEL profit order!')
res = self.spec.cancelOrder(element[profitAction]['orderId'])
if not res:
print(self.spec.getLastErrorMessage())
break
element[profitAction]['status'] = 2
return cascade
def checkOrders(self, cascade = None):
if (cascade is None):
cascade = self.cascade
if (cascade is None):
print('checkOrders. Cascade element not defined')
quit()
for element in cascade:
if self.__isActiveOrder(element['buyOrder']):
status = self.spec.getOrderStatus(element['buyOrder']['orderId'])
if status is False:
print(self.spec.getLastErrorMessage())
break
element['buyOrder']['status'] = status
if self.__isActiveOrder(element['sellOrder']):
status = self.spec.getOrderStatus(element['sellOrder']['orderId'])
if status is False:
print(self.spec.getLastErrorMessage())
break
element['sellOrder']['status'] = status
return cascade
def getProfit(self, cascade = None):
if (cascade is None):
cascade = self.cascade
if (cascade is None):
print('getProfit. Cascade element not defined')
quit()
profit = False
if self.isRevers(cascade):
accepted = 0
for element in cascade:
accepted += round(element['sellOrder']['price'] * element['sellOrder']['operationAmount'] * (100 - self.fee) / 100, self.pricePrecision)
if self.__isCompleteOrder(element['buyOrder']):
used = round(element['buyOrder']['price'] * element['buyOrder']['operationAmount'], self.totalPrecision)
profit = accepted - used
print('Stage: {0}, profit: {1}'.format(element['stage'], profit))
profit = round(profit, self.profitPrecision)
else:
invested = 0
for element in cascade:
invested += round(element['buyOrder']['price'] * element['buyOrder']['operationAmount'], self.totalPrecision)
if self.__isCompleteOrder(element['sellOrder']):
accepted = round(element['sellOrder']['price'] * element['sellOrder']['operationAmount'] * (100 - self.fee) / 100, self.totalPrecision)
profit = accepted - invested
print('Stage: {0}, profit: {1}'.format(element['stage'], profit))
profit = round(profit, self.profitPrecision)
return profit
##
# @brief Brief
#
# @param [in] self Parameter_Description
# @param [in] cascade Parameter_Description
# @param [in] cascadeFileName Parameter_Description
# @return true in success case
#
# @details cancel last sell order and save direct cascade
#
def saveCascade(self, cascade = None, cascadeFileName = None):
if (cascade is None):
cascade = self.cascade
if (cascade is None):
print('saveCascade. Cascade element not defined')
return False
if cascadeFileName is None:
print('saveCascade. Cascade file name not defined')
return False
idx = len(cascade) - 1
if not self.__isActiveOrder(cascade[idx]['sellOrder']):
print('saveCascade. Last save order is not active')
return False
res = self.spec.cancelOrder(cascade[idx]['sellOrder']['orderId'])
if not res:
print('saveCascade. Cancel order error ' + self.spec.getLastErrorMessage())
return False
os.rename(cascadeFileName, cascadeFileName + '.bkp')
return True
##
# @brief Brief
#
# @param [in] self Parameter_Description
# @param [in] cascade Parameter_Description
# @param [in] cascadeFileName Parameter_Description
# @return Return_Description
#
# @details Details
#
def restoreCascade(self, cascade = None, cascadeFileName = None):
if (cascade is None):
cascade = self.cascade
if (cascade is None):
print('saveCascade. Cascade element is not defined')
return False
if cascadeFileName is None:
print('saveCascade. Cascade file name is not defined')
return False
os.rename(cascadeFileName + '.bkp', cascadeFileName)
file = open(cascadeFileName, 'r+')
cascade = json.load(file)
file.close()
idx = len(cascade) - 1
orderId = self.spec.createOrder(cascade[idx]['sellOrder'])
if orderId is False:
print('restoreCascade. Create order error ' + self.spec.getLastErrorMessage())
return False
cascade[idx]['sellOrder']['orderId'] = orderId
if orderId is None:
cascade[idx]['sellOrder']['status'] = 1
else:
cascade[idx]['sellOrder']['status'] = 0
file = open(cascadeFileName, 'w+')
file.write(json.dumps(cascade))
file.close()
return True
def __isActiveOrder(self, order):
if 'orderId' in order and 'status' in order and order['status'] == 0:
return True
return False
def __isCreatedOrder(self, order):
if 'orderId' in order:
return True
return False
def __isCompleteOrder(self, order):
if 'orderId' in order and 'status' in order and order['status'] == 1:
return True
return False
def __init__(self, key = None, secret = None, silent = False): self.spec = Spec(key, secret, silent)
def ATCS_APD(jobs: list,
machines: list,
setup_times: list,
tao: float = 0.5,
R: float = 0.8):
""" ATCS_APD mette insieme l'ATCS index e l'APD, dato che APD tiene conto
in maniera più precisa dei setup times
Args:
jobs (list): [description]
machines (list): [description]
setup_times (list): [description]
tao (float, optional): [description]. Defaults to 0.5.
R (float, optional): [description]. Defaults to 0.8.
"""
# calcolo s come media dei setup times
# calcolo p come media dei processing times
s = Spec.get_setup_time_avg(setup_times)
p = Job.get_processing_time_avg(jobs)
# calcolo parametri di ATCS
K1 = 1.2 * np.log(len(jobs) / len(machines)) - R
#if tao<0.5 or (nano<0.5 and micro>0.5):
if tao < 0.5:
K1 -= 0.5
A2 = 1.8 if tao < 0.8 else 2.0
K2 = tao / (A2 * math.sqrt(s / p))
wi = 1
U = deepcopy(jobs)
print_schedule_d(machines, 2)
# step 0 - Calcolo l'APD per ogni job e setto t=0
apds = APD(U, setup_times)
t = 0
# Step 1 - Prendo la macchina libera al tempo t, poi
# Per ogni job non schedulato:
# a. Calcolo l'ATCS_APD index
# b. Assegno alla macchina selezionata il job con ATCS_APD index maggiore
# c. Imposto t come il loading machine
print_d("Index APD_ATCS:", 3)
while U:
I_i = dict()
# prendo la macchina libera al tempo t
k = Machine.free_machine_at_t(machines, t)
# job_j è l'ultimo job processato dalla macchina selezionata. Serve per il setup time
job_j = k._jobs[-1] if k._jobs else None
# a. calcolo l'ATCS_APD index
for job_u in U:
apd_i = apds[job_u]
I_i[job_u] = ((wi / job_u._processing_time) * math.exp(
-max(job_u._due_date - job_u._processing_time - t, 0) /
(K1 * p)) * math.exp(-Job.get_setup_time(job_j, job_u) /
(K2 * s)) * math.exp(-1 / (apd_i * s)))
I_i = dict(sorted(I_i.items(), key=lambda item: item[1], reverse=True))
# b. assegno alla macchina il job con index maggiore alla macchina
for j in I_i:
# c. imposto il tempo t al loading time
t = k.get_free_time() + Job.get_setup_time(job_j, j)
print_d(j, 3)
k._jobs.append(j)
U.remove(j)
break
print_d("Scheduling APD_ATCS:", 3)
print_schedule_d(machines, 3)
def ATCS(jobs, machines, setup_times, tao=0.9, R=0.2):
""" Calcola l'ATCS index, che tiene conto dei setup times, oltre che
il processing time ed il due date
Args:
jobs (list): [description]
machines (list): [description]
setup_times (list): [description]
tao (float, optional): [description]. Defaults to 0.5.
R (float, optional): [description]. Defaults to 0.8.
"""
# calcolo s come media dei setup times
# calcolo p come media dei processing times
s = Spec.get_setup_time_avg(setup_times)
p = Job.get_processing_time_avg(jobs)
# calcolo parametri di ATCS
K1 = 1.2 * np.log(len(jobs) / len(machines)) - R
#if tao<0.5 or (nano<0.5 and micro>0.5):
if tao < 0.5:
K1 -= 0.5
A2 = 1.8 if tao < 0.8 else 2.0
K2 = tao / (A2 * math.sqrt(s / p))
wi = 1
U = deepcopy(jobs)
print_schedule_d(machines, 2)
# Step 1 - Prendo la macchina libera al tempo t, poi
# Per ogni job non schedulato:
# a. Calcolo l'ATCS index
# b. Assegno alla macchina selezionata il job con ATCS_APD index maggiore
print_d("Index ATCS:", 2)
while U:
I_i = dict()
# prendo la macchina libera al tempo t
k = Machine.first_machine_avaible(machines)
# job_j è l'ultimo job processato dalla macchina selezionata. Serve per il setup time
job_j = k._jobs[-1] if k._jobs else None
# a. calcolo l'ATCS index
for job_u in U:
tmp_machine = deepcopy(k)
tmp_machine._jobs.append(job_u)
t = tmp_machine.get_free_time()
I_i[job_u] = ((wi / job_u._processing_time) * math.exp(
-max(job_u._due_date - job_u._processing_time - t, 0) /
(K1 * p)) * math.exp(-Job.get_setup_time(job_j, job_u) /
(K2 * s)))
I_i = dict(sorted(I_i.items(), key=lambda item: item[1], reverse=True))
# b. assegno alla macchina il job con index maggiore alla macchina
for j in I_i:
print_d(j, 2)
k._jobs.append(j)
U.remove(j)
break
print_d("Scheduling ATCS:", 2)
print_schedule_d(machines, 2)
"""
import config
from spec import Spec
import os.path
key = None
secret = None
if os.path.isfile('../arbitrage.key'):
f = open('../arbitrage.key', 'r')
key = f.readline().strip()
secret = f.readline().strip()
f.close()
curSite = Spec(key, secret)
if not curSite.checkConnection():
quit()
if not curSite.loadTradeConditions():
quit()
if not curSite.generateTradeSequence(config.startCurrency, config.tradeSequence, config.tradeLength):
quit()
if curSite.hasSavedTrades():
selectedTrades = curSite.loadTrades()
key = 0
cont = True
class Scriptor:
"""
The phases of processing:
- Set up data structures and global attributes
- Prepare image specs and organize them
- Launch one thread to initialize image spec (read input image, set up (random) attributes for animation and transition)
- Launch multiple threads to generate frames and store results
- In main thread: wait for results and write to video
- Join video with audio
"""
def generateVideo(self, scriptFile):
with open(scriptFile) as t:
self.rootSpec = Spec(yaml.safe_load(t), None)
rootSpec = self.rootSpec
self.framerate = rootSpec.get('framerate', 30)
self.frameWidth = rootSpec.get('framewidth', 1440)
self.frameHeight = rootSpec.get('frameheight', 1080)
self.outputFrames = rootSpec.get('outputframes')
self.limitFrames = rootSpec.get('limitframes')
random.seed(rootSpec.get('randomseed'))
outputFile = rootSpec.get('outputfile', 'video.mp4')
videoOut = outputFile + '.temp.mp4'
# Initialize data structures
self.imageSpecQueue = queue.Queue()
self.imageFrameQueue = queue.Queue()
self.resultQueue = queue.Queue()
self.prevSpec = None
self.allImageSpecsInitialized = False
# Prepare data structures for processing
images = rootSpec.get('images', [])
self.prepareImageSpecs(images, rootSpec)
# Start one thread to initialize image specs
threading.Thread(target=self.runnableInitImageSpecs).start()
# Start processing image specs by launching worker threads
self.globalFrameN = 0
for _ in range(self.rootSpec.get('threads', 16)):
threading.Thread(target=self.runnableProcessFrame).start()
# In the current thread, wait for and write the results
self.writer = imageio.get_writer(videoOut,
fps=self.framerate,
macro_block_size=8)
self.processResults()
self.writer.close()
# Join audio
audioSpec = rootSpec.getSpec('audio')
if not audioSpec is None:
self.combineVideoWithAudio(audioSpec, videoOut, outputFile)
def prepareImageSpecs(self, images, parentSpec):
""" Walks through the image specs recursively, in order, links them, adds them to the queues.
"""
for item in images:
itemSpec = ImageSpec(item, parentSpec)
subgroup = itemSpec.get('images', None, doRecurse=False)
if subgroup is None:
# Set required variable in prev spec from current spec
if not self.prevSpec is None:
self.prevSpec.nextTransitionDuration = itemSpec.get('transitiontime', 0)
# Link and remember previous
itemSpec.prevSpec = self.prevSpec
self.prevSpec = itemSpec
# Put in queues
self.imageSpecQueue.put(itemSpec)
self.resultQueue.put(itemSpec)
else:
# Recurse
self.prepareImageSpecs(subgroup, itemSpec)
def runnableInitImageSpecs(self):
# Initialize image specs while they are available
# (the queue is pre-filled, so when it's empty, we're done)
imageSpec = getFromQueue(self.imageSpecQueue)
while not imageSpec is None:
self.initializeImageSpec(imageSpec)
# Wait with initializing next image spec.
# (we don't want to initialize and load too early, to limit memory usage,
# but we also don't want to load too late, because it will block the threads,
# so start loading when we have less than a certain amount of frames to process)
while self.imageFrameQueue.qsize() > 60:
time.sleep(0.1)
imageSpec = getFromQueue(self.imageSpecQueue)
# Flag that allows frame processing threads to finish if there are no more frames
self.allImageSpecsInitialized = True
print("finished processing image specs")
def initializeImageSpec(self, imageSpec):
# Read image
inputFileName = imageSpec.get('file')
#assert not inputFileName is None, 'No input file specified'
if inputFileName is None:
npImCurrent = np.zeros((self.frameHeight, self.frameWidth, 3), dtype='uint8')
else:
npImCurrent = imageio.imread('./input/%s' % inputFileName)
# Set up transition
#TODO: relfect: transitionType = transitionSpec.get('type', 'blend')
imageSpec.transition = BlendTransition()
# Set up animation
#animationType = animationSpec.get(Props.IMAGE_ANIMATION_TYPE)
#TODO: use reflection to instantiate:
imageSpec.animation = PanZoomAnimation(npImCurrent, imageSpec)
imageSpec.duration = duration = imageSpec.get('duration', 2.0)
nframes = int(duration * self.framerate)
imageSpec.frames = [None] * nframes
for i in range(0, nframes):
self.imageFrameQueue.put((imageSpec, i))
def runnableProcessFrame(self):
imageFrame = getFromQueue(self.imageFrameQueue)
while (not imageFrame is None) or (not self.allImageSpecsInitialized):
# Either we have an image to process or we have to wait for one
if not imageFrame is None:
(imageSpec, frameNr) = imageFrame
self.processFrame(imageSpec, frameNr)
else:
time.sleep(0.1)
imageFrame = getFromQueue(self.imageFrameQueue)
print("finished processing frames")
def processFrame(self, imageSpec, i):
prevSpec = imageSpec.prevSpec
transitionDuration = imageSpec.get('transitiontime', 0.5)
duration = imageSpec.duration
nextTransitionDuration = imageSpec.nextTransitionDuration
animation = imageSpec.animation
transition = imageSpec.transition
print("processing %s frame %d/%d" % (imageSpec.get('file'), i + 1, len(imageSpec.frames)))
# Animate image
animationT = self.getTForFrame(i, duration + nextTransitionDuration, self.framerate)
npIm1 = animation.animate(animationT)
# Transition
transitionT = 1.0 if (transitionDuration == 0) else i / (transitionDuration * self.framerate)
if transitionT < 1.0 and not prevSpec is None:
# Animate previous image
animationT = self.getTForFrame(prevSpec.duration * self.framerate + i,
prevSpec.duration + transitionDuration, self.framerate)
npIm0 = prevSpec.animation.animate(animationT)
# Combine transition images
npResult = transition.processTransition(npIm0, npIm1, transitionT)
else:
npResult = npIm1
# Put result in list to be written
imageSpec.frames[i] = npResult
def processResults(self):
# self.resultQueue has been prefilled (to keep results in the correct order),
# so we just need to keep going until it is empty
currentResult = getFromQueue(self.resultQueue)
while not currentResult is None:
# Wait for initialization
while currentResult.frames is None:
time.sleep(0.1)
# Wait for and process each result frame in order
for i in range(len(currentResult.frames)):
# Wait for result
while currentResult.frames[i] is None:
time.sleep(0.1)
# Process result
self.writeResultImage(currentResult.frames[i])
# Clean up
currentResult.frames[i] = None
# Clean up unused references to free memory
if not currentResult.prevSpec is None:
# Clean up finished spec so memory can be released
currentResult.prevSpec.animation = None
currentResult.prevSpec.transition = None
currentResult.prevSpec = None
currentResult = getFromQueue(self.resultQueue)
def writeResultImage(self, image):
# Write frame to video
self.writer.append_data(image)
# Write frame to image if set up
if not self.outputFrames is None:
imageio.imwrite(self.outputFrames % self.globalFrameN, image)
self.globalFrameN += 1
# Scales the frameNumber to the current position in the animation to a fraction
# between 0 and 1.
# totalDuration should include the animation duration for the current image and the
# transition duration to the next image
def getTForFrame(self, frameNumber, totalDuration, frameRate):
return frameNumber / (totalDuration * frameRate)
def combineVideoWithAudio(self, audioSpec, videoIn, videoOut):
def maybe(option, key, spec):
value = spec.get(key)
return [option, str(value)] if not value is None else []
audioIn = audioSpec.get('file')
cmd_out = ['ffmpeg',
'-y',
'-i', videoIn,
*maybe('-ss', 'audiooffset', audioSpec),
*maybe('-itsoffset', 'videooffset', audioSpec),
'-i', audioIn,
#'-c', 'copy',
'-map', '0:v',
'-map', '1:a',
'-shortest',
'-filter:a', 'afade=t=out:st=144:d=8',
videoOut]
pipe = subprocess.Popen(cmd_out)
pipe.wait()
