def update_db_time_range(symbol):
with classes.SQLite(config.DB_SYMBOLS_PATH, 'update_db_time_range:',
None) as cur:
# Fill the DB first, this code thinks it will return something anyway
exec_string = f"SELECT [DateTime], Close From {symbol} ORDER BY [DateTime]"
dates = db_time_ranges[symbol]
date_start = None
if dates[0] is None:
first_row = cur.execute(exec_string).fetchone()
result = first_row[0]
last_price = first_row[1]
date_start = helper.str_to_utc_datetime(result, "UTC",
config.DB_DATE_FORMAT)
else:
date_start = dates[0]
exec_string = f"{exec_string} DESC"
result = cur.execute(exec_string).fetchone()
date_end = helper.str_to_utc_datetime(result[0], "UTC",
config.DB_DATE_FORMAT)
last_price = result[1]
db_time_ranges[symbol] = (date_start, date_end, last_price)
dates = f"symbol:{symbol}, date_start: {date_start}, date_end: {date_end}"
logging.info(dates)
def convert_history_json(file_in, timezone_local):
json = config.get_json(file_in)
json_array = json.get("messages")
channel_id = json.get("id")
if json_array is None or channel_id is None:
print("Cannot parse Telegram history json")
return
messages_list = list()
for message in json_array:
msg_props = dict()
msg_props["id"] = message["id"]
msg_props["date"] = helper.str_to_utc_datetime(
message["date"], timezone_local, config.ISO_LOCAL_DATE_FORMAT).isoformat()
message_item = message.get("text")
if message_item is not None:
msg_props["text"] = str(message_item)
reply_to_message_id = message.get("reply_to_message_id")
if reply_to_message_id is not None:
msg_props["reply_to_msg_id"] = reply_to_message_id
messages_list.append(msg_props)
out_path = os.path.join(
config.CHANNELS_HISTORY_DIR, f"{channel_id}.json")
config.set_json(out_path, messages_list)
print(f"Saved as {out_path}")
def parse_alphavantage(symbol, symbol_last_datetime):
lag = get_lag_mins(symbol_last_datetime)
url = symbol_api_mapping[symbol]
if lag > API_EXTENDED_POLL_THRESHOLD_MIN:
url = f"{url}&outputsize=full"
req = requests.get(url)
content = req.text
price_data = json.loads(content)
meta = get_array_item_contains_key(price_data, "meta")
timezone = get_array_item_contains_key(meta, "time zone")
price_data = get_array_item_contains_key(price_data, "series")
sorted_items = sorted(price_data.keys())
for price_item in sorted_items:
utc_date = helper.str_to_utc_datetime(
price_item, timezone, POLL_INPUT_FORMAT)
open_ = price_item[1]
close = price_item[2]
high = price_item[3]
low = price_item[4]
yield (utc_date, open_, high, low, close)
def should_wait(date_str):
last_date = helper.str_to_utc_datetime(date_str, "UTC", ISO_DATE_FORMAT)
now = datetime.datetime.utcnow()
next_collect_date = None
weekday = now.weekday()
if weekday == 5 or weekday == 6: # weekend
return True
else:
next_collect_date = last_date + datetime.timedelta(seconds=delay_sec)
if next_collect_date > utc.localize(now):
return True
return False
def import_csv(symbol, input_file, symbol_last_datetime):
# symbol_last_datetime = db_time_ranges[symbol][1] for ex.
sql_connection = sqlite3.connect(config.DB_SYMBOLS_PATH)
cur = sql_connection.cursor()
count = 0
high_prev: decimal.Decimal = None
low_prev: decimal.Decimal = None
open_prev: decimal.Decimal = None
rounded_min_datetime_prev: datetime.datetime = None
# parse ordered datetimes only. You can export them from MetaTrader
for line in fileinput.input([input_file]):
array = str(line).split("\t")
if len(array) < 4:
continue
bid = helper.str_to_decimal(array[2])
if bid is None:
continue
try:
date = helper.str_to_utc_datetime(
f'{array[0]}T{array[1]}', DT_INPUT_TIMEZONE, DT_INPUT_FORMAT)
if date is None or symbol_last_datetime > date:
continue
rounded_min_datetime = date - \
datetime.timedelta(seconds=date.second) - \
datetime.timedelta(microseconds=date.microsecond)
iso_date = rounded_min_datetime.isoformat(" ")
high_ = bid
low_ = bid
open_ = bid
exec_string = None
if rounded_min_datetime_prev == rounded_min_datetime:
if bid < high_prev:
high_ = high_prev
if bid > low_prev:
low_ = low_prev
open_ = open_prev
exec_string = f"UPDATE {symbol} SET High={high_}, Low={low_}, Close={bid} WHERE [DateTime]='{iso_date}'"
else:
exec_string = f"INSERT INTO {symbol} VALUES ('{iso_date}',{bid},{bid},{bid},{bid}) ON CONFLICT(DateTime) DO UPDATE SET Close=excluded.Close"
cur.execute(exec_string)
count += 1
if count % COMMIT_BATCH_ROW_COUNT == 0:
sql_connection.commit()
print("Count %s, symbol %s" % (count, symbol))
rounded_min_datetime_prev = rounded_min_datetime
high_prev = high_
low_prev = low_
open_prev = open_
except Exception as ex:
print("Error %s" % ex)
continue
sql_connection.commit()
sql_connection.close()
def analyze_channel(channel_id):
logging.info('analyze_channel, channel id: %s', channel_id)
out_path = os.path.join(config.CHANNELS_HISTORY_DIR, f"{channel_id}.json")
messages = None
try:
messages = config.get_json(out_path)
except Exception as ex:
logging.error('analyze_channel: %s, error: %s', ex,
traceback.format_exc())
with classes.SQLite(config.DB_STATS_PATH, 'analyze_channel_error:',
lock) as cur:
update_string = f"UPDATE Channel SET HistoryLoaded = 0 WHERE Id={channel_id}"
cur.execute(update_string)
return
if messages is None or len(messages) < 1:
logging.info('analyze_channel: no data from %s', out_path)
return
ordered_messges = sorted(messages, key=lambda x: x["id"], reverse=False)
min_channel_date = helper.str_to_utc_datetime(ordered_messges[0]["date"])
max_channel_date = helper.str_to_utc_datetime(
ordered_messges[len(ordered_messges) - 1]["date"])
for symbol in signal_parser.symbols_regex_map:
min_date = db_poll.db_time_ranges[symbol][0]
max_date = db_poll.db_time_ranges[symbol][1]
if (min_channel_date > min_date):
min_date = min_channel_date
if (max_channel_date < max_date):
max_date = max_channel_date
min_date_rounded_minutes = min_date - datetime.timedelta(
seconds=min_date.second, microseconds=min_date.microsecond)
max_date_rounded_minutes = max_date - datetime.timedelta(
seconds=max_date.second, microseconds=max_date.microsecond)
while not WAIT_EVENT_OUTER.is_set():
if is_theads_busy():
WAIT_EVENT_INNER.wait(STATS_ANALYZE_LOOP_GAP_SEC)
else:
logging.info(
'analyze_channel: id: %s, symbol: %s, start: %s, end: %s',
channel_id, symbol, min_date_rounded_minutes,
max_date_rounded_minutes)
process_channel_typle = (ordered_messges, symbol,
min_date_rounded_minutes,
max_date_rounded_minutes, channel_id)
atomic_increment()
pool.apply_async(signal_parser.analyze_channel_symbol,
process_channel_typle,
callback=write_db)
break
if WAIT_EVENT_OUTER.is_set():
return
async def collect(stop_flag: classes.StopFlag):
total = last_id + 1 + length
log_every = length / 2
log_count = 0
for current_number in range(last_id + 1, total, STEP):
log_count = log_count + STEP
if log_count > log_every:
log_count = 0
logging.info("collecting... {0}%".format(
100 * (total - current_number) / total))
if stop_flag.Value:
return
try:
url = f"{main_url_part}{current_number}{url_tail}"
result = requests.get(url, headers=chrome_headers)
is_404 = result.status_code == 404
if is_404:
continue
if result.ok:
content = json.loads(result.text)
name = content['name']
published_at = content['published_at']
published_at_date = helper.str_to_utc_datetime(
published_at, "UTC", ISO_DATE_FORMAT)
view_url = f"{view_url_part}{current_number}{url_tail}"
logging.info(
f"video \"{name}\" ({published_at}) found: {view_url}")
has_video = True
out_file = f"{current_number}.mp4"
try:
ydl_opts = {"outtmpl": out_file}
with youtube_dl.YoutubeDL(ydl_opts) as ydl:
ydl.download([view_url])
except Exception as ex:
logging.info(f"on grab error: {ex}")
has_video = False
text_msg = f"\"{name}\" - {published_at}"
async with TelegramClient(SESSION, config.api_id,
config.api_hash) as client:
for send_id in send_ids:
if has_video:
await client.send_file(send_id, out_file)
await client.send_message(
send_id,
f"{text_msg}\n{view_url}",
silent=True)
else:
await client.send_message(
send_id, f"{text_msg}\n{view_url}")
if has_video:
os.remove(out_file)
to_save = dict()
to_save['id'] = current_number
to_save['published_at'] = published_at
to_save['name'] = content['name']
to_save['url'] = view_url
result_list.insert(0, to_save)
config.set_json(FILE_DB, result_list)
config_collector["last_id"] = current_number
config_collector["last_date"] = published_at
config.set_json(COLLECTOR_CONFIG, config_collector)
got_collected = True
load_cfg()
if should_wait(published_at):
return
else:
logging.info(
f"resp_code: {result.status_code}, id: {current_number}, sleep for {ON_ERROR_SLEEP_SEC} sec"
)
await asyncio.sleep(ON_ERROR_SLEEP_SEC)
except Exception as ex:
logging.info(f"collector error: {ex}")
await asyncio.sleep(ON_ERROR_SLEEP_SEC)
def analyze_channel_symbol(ordered_messges, symbol, min_date, max_date,
channel_id):
symbol_regex = symbols_regex_map[symbol]
order_book = list()
min_date_str = min_date.strftime(config.DB_DATE_FORMAT)
max_date_str = max_date.strftime(config.DB_DATE_FORMAT)
symbol_data = None
exec_string = f"SELECT [DateTime], High, Low, Close FROM {symbol} WHERE [DateTime] BETWEEN '{min_date_str}' AND '{max_date_str}' ORDER BY [DateTime]"
with classes.SQLite(config.DB_SYMBOLS_PATH, 'analyze_channel_symbol, db:',
None) as cur:
symbol_data = cur.execute(exec_string).fetchall()
if symbol_data is None or len(symbol_data) == 0:
logging.info('analyze_channel_symbol: no data for symbol %s', symbol)
min_date_str_iso = min_date.strftime(config.ISO_DATE_FORMAT)
max_date_str_iso = max_date.strftime(config.ISO_DATE_FORMAT)
filtered_messages = list(
filter(
lambda x: x["date"] >= min_date_str_iso and x["date"] <=
max_date_str_iso, ordered_messges))
symbol_data_len = len(symbol_data)
i = 0
current_date_str = None
next_date_str = None
logging.info('analyze_channel_symbol: setting orders... %s', symbol)
for symbol_value in symbol_data:
if i < symbol_data_len - 1:
next_value = symbol_data[i + 1]
else:
break
i += 1
current_date_str = helper.str_to_utc_datetime(
symbol_value[0], input_format=config.DB_DATE_FORMAT).isoformat()
next_date_str = helper.str_to_utc_datetime(
next_value[0], input_format=config.DB_DATE_FORMAT).isoformat()
orders_open = list(filter(lambda x: x["is_open"] is True, order_book))
found_messages = list(
filter(
lambda x: x["date"] >= current_date_str and x["date"] <
next_date_str, filtered_messages))
has_messages_in_min = len(found_messages) > 0
has_orders_open = len(orders_open) > 0
if not has_messages_in_min and not has_orders_open:
continue
value_high = symbol_value[1]
value_low = symbol_value[2]
value_close = symbol_value[3]
if has_orders_open:
update_orders(next_date_str, value_high, value_low, value_close,
orders_open)
for msg in found_messages:
id_ = msg["id"]
reply_to_message_id = msg.get("reply_to_msg_id")
is_reply = reply_to_message_id is not None
target_orders = None
if is_reply:
reply_sources = list(
filter(lambda x: x["id"] == reply_to_message_id,
orders_open))
target_orders = reply_sources
else:
target_orders = orders_open
string_to_orders(msg, symbol_regex, target_orders, next_date_str,
value_close, order_book)
return (order_book, symbol, channel_id)