def imagenet1000_classidx_to_label(self, class_idx: int) -> str:
try:
map = self.imagenet1000_classidx_label_dict()
return map[str(class_idx)]
except Exception as e:
logger.exception(e)
return "Class Not Found"
def classify_image(event, context):
try:
picture, filename = utils.get_image_from_event(event)
img_net = ImageNetHelper("mobilenet_v2")
img_net.load_model(S3_BUCKET)
picture_tensor = img_net.transform_image(picture.content)
prediction_idx = img_net.get_prediction(picture_tensor)
prediction_label = img_net.imagenet1000_classidx_to_label(
prediction_idx)
return {
"statusCode":
200,
"headers":
headers,
"body":
json.dumps({
"file": filename,
"predicted": (prediction_idx, prediction_label),
}),
}
except Exception as e:
logger.exception(e)
return {
"statusCode": 500,
"headers": headers,
"body": json.dumps({"error": repr(e)}),
}
def RecvFromSever(self) -> Tuple[Dict, int]:
sockdata: str = ""
try:
sockdata = self.socket.recvfrom(UDP_MSGLEN)[0].decode("utf-8")
if self.verbose:
print("\nrecv", sockdata)
except Exception as e:
logger.exception(f"{e}")
return dict(), SERVERCODE_TIMEOUT
if "***restart***" in sockdata:
logger.info(f"server restarted on port: {self.port}")
return dict(), SERVERCODE_RESTART
elif "***shutdown***" in sockdata:
logger.info(f"server shutdown on port: {self.port}.")
return dict(), SERVERCODE_SHUTDOWN
elif not sockdata:
logger.warning(f"server no response on port: {self.port}")
return dict(), SERVERCODE_NORESPONSE
elif "***identified***" in sockdata:
logger.info(f"client identified connected on port: {self.port}")
# * always delay few secs after starting round maybe ??
time.sleep(self.delay)
return dict(), SERVERCODE_IDENTIFIED
else:
if self.verbose:
logger.info(f"recv: {sockdata}")
return ServerState.Deserialize(sockdata), SERVERCODE_OBSERVATION
return True
def wrapper(*args, **kwargs):
response = None
try:
response = function(*args, **kwargs)
except exception:
logger.exception(message)
return response
def imagenet1000_classidx_label_dict(self) -> dict:
try:
with open("data/imagenet1000_clsidx_to_labels.json", "r") as f:
map = json.loads(f.read())
return map
except Exception as e:
logger.exception(e)
return None
def save_pretrained_model(self):
try:
model = self._model_func(pretrained=True)
model.eval()
# trace model with dummy input
traced_model = torch.jit.trace(model, torch.randn(1, 3, 224, 224))
traced_model.save(f"./models/{self._model_name}.pt")
except Exception as e:
logger.exception(e)
def get_picture_filename(picture):
try:
filename = picture.headers[b"Content-Disposition"].decode().split(
":")[1].split("-")[1]
if 4 > len(filename):
filename = picture.headers[b"Content-Disposition"].decode().split(
":")[2].split("-")[1]
except Exception as e:
filename = "not-found"
logger.exception(e)
return filename
def transform_image(self, image_bytes):
try:
transformations = transforms.Compose([
transforms.Resize(255),
transforms.CenterCrop(224),
transforms.ToTensor(),
transforms.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225]),
])
image = Image.open(io.BytesIO(image_bytes))
return transformations(image).unsqueeze(0)
except Exception as e:
logger.exception(e)
raise (e)
def decrypt(self):
if encrypted:
cipher = AES.new(
getpass.getpass(
'Input decryption password (string will not show)'))
try:
self.api_key = ast.literal_eval(self.api_key) if type(
self.api_key) == str else self.api_key
self.api_secret = ast.literal_eval(self.api_secret) if type(
self.api_secret) == str else self.api_secret
except Exception:
logger.exception(Exception)
pass
self.api_key = cipher.decrypt(self.api_key).decode()
self.api_secret = cipher.decrypt(self.api_secret).decode()
else:
raise ImportError('"pycrypto" module has to be installed')
def load_model(self, s3_bucket: str):
try:
# s3 = boto3.session.Session(profile_name='eva4p2').client("s3") # noqa
s3 = boto3.client("s3")
model_path = os.path.join("artifacts/models",
f"{self._model_name}.pt")
obj = s3.get_object(Bucket=s3_bucket, Key=model_path)
logger.info("Creating Byte Stream")
bytestream = io.BytesIO(obj["Body"].read())
logger.info("Loading model")
model = torch.jit.load(bytestream)
logger.info("Model Loaded...")
logger.info(model)
logger.info(model.code)
self._model = model
except Exception as e:
logger.exception(e)
raise (e)
def job(query_hash):
logger.debug('Starting to process %s' % query_hash)
try:
start_time = timeit.default_timer()
events_pushed = push_data_from_query_hash(query_hash)
logger.info(
'Done with the process of %s : %s events, after %1.2fs',
query_hash,
len(events_pushed),
timeit.default_timer() - start_time,
)
except: # noqa: E722
logger.exception(
f'Threats for query hash {query_hash} failed to be retrieved and injected into misp'
)
raise
finally:
jobs_running.remove(query_hash)
def get_historical_data(self, market, period, unit):
"""
Queries the historical data in the form of a list
:param market: String literal for the market (ex: BTC-LTC)
:type market: str
:param period: Number of periods to query
:type period: int
:param unit: Ticker interval (one of: 'oneMin', 'fiveMin', 'thirtyMin', 'hour', 'week', 'day', and 'month')
:type unit: str
:return: List adapted from Bittrex JSON response
:rtype : list
"""
request_url = 'https://bittrex.com/Api/v2.0/pub/market/GetTicks?marketName={}&tickInterval={}'.format(
market, unit)
try:
historical_data = requests.get(request_url,
headers={
"apisign":
hmac.new(
self.api_secret.encode(),
request_url.encode(),
hashlib.sha512).hexdigest()
}).json()
except json.decoder.JSONDecodeError as exception:
logger.exception(exception)
return []
if not historical_data['success']:
if historical_data['message'] == 'INVALID_MARKET':
logger.warning(
'The {} market is currently not available on Bittrex'.
format(market))
return []
return historical_data['result'][-period:]
def get_message(text):
global mood_value
try:
if len(text) > 0:
text_sentiment = sentiment_from_text(text)
if text_sentiment == None:
return None
total_sentiment = (sentiment_from_mood['sentiment'] *
text_sentiment + sentiment_from_mood['mood'] *
mood_value) / sentiment_from_mood['sum']
logger.debug(['total sentiment', total_sentiment])
result_sentiment = chose_message(total_sentiment)
set_mood((mood_from_sentiment['sentiment'] * text_sentiment +
mood_from_sentiment['mood'] * mood_value) /
mood_from_sentiment['sum'])
else:
result_sentiment = chose_message(mood_value)
except Exception as e:
logger.exception(e)
return None
if len(messages) > 0:
return random.choice(messages[result_sentiment])
else:
return None
current_price = get_current_price(coin_pair)
profit_margin = Database_List[strategy_index].get_simulated_profit_margin(coin_pair, current_price)
if profit_margin >= 2.5:
print(print_str.format(strategy_index, coin_pair, round(rsi), round(profit_margin),
Messenger.generate_bittrex_URL(coin_pair)))
Database_List[strategy_index].simulate_sell(coin_pair, current_price, rsi, day_volume)
if __name__ == '__main__':
def analyse_buys():
for strategy_index in range(number_of_strategies):
if len(Database_List[strategy_index].simulated_trades['trackedCoinPairs']) < 1:
for coin_pair in btc_coin_pairs:
buy_strategy(coin_pair, strategy_index)
def analyse_sells():
for strategy_index in range(number_of_strategies):
for coin_pair in Database_List[strategy_index].simulated_trades['trackedCoinPairs']:
sell_strategy(coin_pair, strategy_index)
btc_coin_pairs = get_markets('BTC')
print('Tracking {} Bittrex markets'.format(len(btc_coin_pairs)))
while True:
try:
analyse_buys()
analyse_sells()
except Exception:
logger.exception(Exception)
args['save_path_optimizer_encoder'], map_location="cpu")
encoder_optimizer.load_state_dict(encoder_optimizer_state)
encoder_optimizer = create_correct_state_dict(encoder_optimizer)
decoder_optimizer = optim.Adam(decoder.parameters(),
lr=learning_rate *
decoder_learning_ratio)
decoder_optimizer_state = torch.load(
args['save_path_optimizer_decoder'], map_location="cpu")
decoder_optimizer.load_state_dict(decoder_optimizer_state)
decoder_optimizer = create_correct_state_dict(decoder_optimizer)
from_scratch = False
logger.info('Continue training')
except Exception as e:
logger.exception(e)
logger.info('From scratch')
pass
if USE_CUDA:
encoder = encoder.cuda()
decoder = decoder.cuda()
print_loss_total = 0
save_every = 500
batch_n = 0
epoch = 0
print_every = 125
print_every_val = 125
start = time.time()