def authenticate(username, password):
url = base_url + "login"
payload = {
'client_id': username,
'client_secret': password
}
# Request to server
# For real cert
# r = requests.get(url, data=data, verify=True)
# For self cert
try:
r = requests.post(url, data=payload, verify=cert)
except Exception as e:
log.print_exception(e)
log.print_error("error", "connection failure")
return False
# Parse result
try:
response = json.loads(r.text)
except Exception as e:
log.print_exception(e)
log.print_error("authentication failure", "failed to decode server message '%s'" % (r.text))
return False
if response.get("status", False) and response.get("token", None) != None:
# Authentication success
global token
global client_id
token = response.get("token")
client_id = username
return True
else:
# Authentication failure
log.print_error("authentication failure", "wrong username or password")
return False
def logout():
global token
if token == None:
return False
url = base_url + "logout"
data = {"token": token}
#r = requests.get(url, data=data, verify=True)
try:
r = requests.get(url, params=data, verify=cert)
except Exception as e:
log.print_exception(e)
log.print_error("error", "connection failure")
return False
# Parse result
try:
response = json.loads(r.text)
except Exception as e:
log.print_exception(e)
log.print_error("logout failed", "failed to decode server message '%s'" % (r.text))
return False
if response.get("status", False):
token = None
# Logout successful
return True
else:
# Logout failure
log.print_error("logout failed", "failed to logout '%s'" % (r.text))
print("d")
return False
def share(recipient, data):
"""
Upload the file to server.
"""
url = base_url+"share"
print(token)
print(recipient)
print(data)
data = {
'token': token,
'recipient': recipient,
'data': data
}
try:
r = requests.post(url, data=data, verify=cert)
except Exception as e:
log.print_exception(e)
log.print_error("error", "connection failure")
return False
# Parse result
try:
response = json.loads(r.text)
print(response)
except Exception as e:
log.print_exception(e)
log.print_error("authentication failure", "failed to decode server message '%s'" % (r.text))
return False
if response.get("status"):
return True
else:
return False
def get_vix(dates):
try:
URL = "https://www.nseindia.com/products/dynaContent/equities/indices/hist_vix_data.jsp?&fromDate=08-Oct-2018&toDate=31-Oct-2018"
url = "https://www.nseindia.com/products/dynaContent/equities/indices/hist_vix_data.jsp?&fromDate={start}&toDate={end}"
dfs = []
for x in dates.iterrows():
urlg = url.format(
start=x[1][0].strftime("%d-%b-%Y"), end=x[1][1].strftime("%d-%b-%Y")
)
print(urlg)
dfi = indexdb.get_csv_data(urlg, indexdb.vix_cols)
if dfi is not None:
dfs.append(dfi)
if len(dfs) > 1:
dfo = pd.concat(dfs)
elif len(dfs) == 1:
dfo = dfs[0]
else:
dfo = None
return dfo
except Exception as e:
print(url)
print(urlg)
print_exception(e)
return None
def download(filename_rand, saveas=None, shared_by=None):
'''
Download and save the file from server.
'''
if saveas == None:
saveas = filename_rand
url = base_url+"download"
data = {
'token': token,
'file': filename_rand
}
if shared_by != None:
data['shared_by'] = shared_by
#r = requests.get(url, data=data, verify=True)
try:
r = requests.get(url, params=data, verify=cert)
except Exception as e:
log.print_exception(e)
log.print_error("error", "connection failure")
return False
if r.content == b'404 page not found\n':
return False
try:
with open(saveas, "wb") as f:
f.write(r.content)
return True
except Exception as e:
log.print_exception(e)
log.print_error("IO error", "cannot open file '%s'" % (filename_rand))
raise
def delete(filename_rand):
"""
Upload the file to server.
"""
url = base_url
data = {
'token': token,
'file': filename_rand
}
try:
r = requests.delete(url, params=data, verify=cert)
except Exception as e:
log.print_exception(e)
log.print_error("error", "connection failure")
return False
# Parse result
try:
response = json.loads(r.text)
except Exception as e:
log.print_exception(e)
log.print_error("authentication failure", "failed to decode server message '%s'" % (r.text))
return False
if response.get("status"):
return True
else:
return False
def upload(filename_rand):
"""
Upload the file to server.
"""
url = base_url+"upload"
data = {'token': token}
files = {'document': open(filename_rand, 'rb')}
#r = requests.post(url, data=data, files=files, verify=True)
try:
r = requests.post(url, data=data, files=files, verify=cert)
except Exception as e:
log.print_exception(e)
log.print_error("error", "connection failure")
return False
files['document'].close()
# Parse result
try:
response = json.loads(r.text)
except Exception as e:
log.print_exception(e)
log.print_error("authentication failure", "failed to decode server message '%s'" % (r.text))
return False
if response.get("status"):
return True
else:
return False
def updateIndexData(dates, index="NIFTY%2050", symbol="NIFTY"):
try:
url = "https://www.nseindia.com/products/dynaContent/equities/indices/historicalindices.jsp?indexType={index}&fromDate={start}&toDate={end}"
dfs = []
for x in dates.iterrows():
urlg = url.format(
index=index,
start=x[1][0].strftime("%d-%m-%Y"),
end=x[1][1].strftime("%d-%m-%Y"),
)
print(urlg)
dfi = indexdb.get_csv_data(urlg, indexdb.idx_cols)
if dfi is not None:
dfs.append(dfi)
if len(dfs) > 1:
dfo = pd.concat(dfs)
elif len(dfs) == 1:
dfo = dfs[0]
else:
dfo = None
if dfo is not None:
dfo = dfo.rename(columns=indexdb.idx_col_rename)
dfo["SYMBOL"] = symbol
return dfo
except Exception as e:
print(urlg)
print_exception(e)
return None
def updateIndex_upto_date():
try:
dates = indexdb.get_next_update_start_date("idx")
if dates is not None:
indexdb.update_index_for_dates(dates)
else:
print("Nothing to update for index...")
except Exception as e:
print_exception(e)
def getHistoricalNiftyAndBankNifty():
try:
if indexdb.check_table_exists("idx"):
idb = pd.HDFStore("indexdb.hdf")
del idb["idx"]
idb.close()
dates = indexdb.get_dates(start="1994-1-1")
indexdb.update_index_for_dates(dates)
except Exception as e:
print_exception(e)
def updateHistoricFNOBhavData(end_date):
"""
Do not call this function as this function tries to update the db since 2000-6-12
"""
try:
df = pd.bdate_range(start="2000-6-12", end=end_date).sort_values(
ascending=False
)
indexdb.updateFNOBhavData_for_given_dates(df)
except Exception as e:
print_exception(e)
def ui_share():
if not client_conn.is_login():
print("Please login first")
return
# Enter recipient username
print("Invite people (username): ", end='')
recipient = input().strip()
# Recipient's email
recv_email = None
print("Recipient's email address: ", end='')
recv_email = input().strip()
# Get target's public key
choice = None
while choice != "1" and choice != "2":
print("Obtain the recipent's public key:")
print(" 1) Download from Hong Kong Post")
print(" 2) Input from file (debug)")
print("Choice [1,2]: ", end='')
choice = input().strip()
public_key = None
try:
if choice == "1":
# Download from HK Post
public_key = rsa.get_cert(recv_email, True)
sender = "*****@*****.**"
if choice == "2":
# Import from file
sender = "*****@*****.**"
filename = "key/public_key.pem"
public_key = rsa.load_public_cert_from_file(filename)
except Exception as e:
log.print_exception(e)
log.print_error("error", "failed to load cert")
return
# Get user's private key to signoff
if os.path.isfile("/home/star/.ssh/me.key.pem2"):
private_key = rsa.load_private_cert_from_file("/home/star/.ssh/me.key.pem2")
else:
private_key = rsa.load_private_cert_from_file("key/private_key.pem")
# Encrypt the filelist record
print("File to share: ", end='')
filename = input()
record = filelist.export_record(filename, sender, recv_email, public_key, private_key)
if record == None:
print("Failed to share file")
return
# Send to server
client_conn.share(recipient, record)
def get_share():
"""
Upload the file to server.
"""
url = base_url+"listshare"
data = {
'token': token,
}
data = {"token": token}
try:
r = requests.get(url, params=data, verify=cert)
except Exception as e:
log.print_exception(e)
log.print_error("error", "connection failure")
return False
# Parse result
try:
response = json.loads(r.text)
print(response)
except Exception as e:
log.print_exception(e)
log.print_error("authentication failure", "failed to decode server message '%s'" % (r.text))
return False
if not response.get("status"):
return False
if response["records"] == None:
return False
for i in response["records"]:
sender = i['Sender']
record = i['Record']
print("Sender:",sender)
print("Record:",record)
try:
if os.path.isfile("/home/star/.ssh/me.key.pem2"):
public_key = rsa.get_cert("*****@*****.**")
else:
public_key = rsa.load_public_cert_from_file("key/public_key.pem")
if os.path.isfile("/home/star/.ssh/me.key.pem2"):
private_key = rsa.load_private_cert_from_file("/home/star/.ssh/me.key.pem2")
else:
private_key = rsa.load_private_cert_from_file("key/private_key.pem")
share = filelist.import_record(record, public_key, private_key)
print(share)
if share != None:
filelist.append_share(share['filename_ori'], share['filename_rand'],
share['key'], share['iv'], share['tag'], sender)
except:
print("Failed to decrypt message")
pass
def load_public_cert_from_file(filename):
"""
Load pem public key from file
"""
try:
with open(filename, "rb") as key_file:
public_key = serialization.load_pem_public_key(
key_file.read(),
backend=default_backend()
)
return public_key
except Exception as e:
log.print_exception(e)
log.print_error("error", "failed to open file '%s'" % (r.text))
return False
def getHistoricalVix():
try:
if not indexdb.check_table_exists("vix"):
dates = indexdb.get_dates(start="2007-1-1")
dfn = indexdb.get_vix(dates)
dfn = dfn.rename(columns=indexdb.vix_col_rename)
dfn.to_hdf(
"indexdb.hdf",
"vix",
mode="a",
append=True,
format="table",
data_columns=True,
)
print(f"Done vix")
except Exception as e:
print_exception(e)
def updateFNOBhavData_upto_date():
try:
dfd = pd.read_hdf(
"indexdb.hdf",
"fno",
where="SYMBOL==NIFTY & INSTRUMENT==FUTIDX",
columns=["TIMESTAMP"],
)
dfd = dfd.sort_values("TIMESTAMP", ascending=False).head(1)
df = pd.bdate_range(
start=dfd["TIMESTAMP"].iloc[0], end=date.today(), closed="right"
)
if len(df) > 0:
indexdb.updateFNOBhavData_for_given_dates(df)
print("Done updating FNO...")
else:
print("Nothing to update for FNO...")
except Exception as e:
print_exception(e)
def get_fno_csv_data(dt):
action_url = "https://www.nseindia.com/ArchieveSearch?h_filetype=fobhav&date={date}§ion=FO"
download_url = "https://www.nseindia.com/content/historical/DERIVATIVES/{year}/{month}/fo{date}bhav.csv.zip"
url = action_url.format(date=dt.strftime("%d-%m-%Y"))
urlp = requests.get(url, headers=indexdb.headers)
if not "No file found" in urlp.content.decode():
urls = download_url.format(
year=dt.year,
month=dt.strftime("%b").upper(),
date=dt.strftime("%d%b%Y").upper(),
)
urlap = requests.get(urls, headers=indexdb.headers)
if urlap.reason == "OK":
byt = BytesIO(urlap.content)
zipfl = zipfile.ZipFile(byt)
zinfo = zipfl.infolist()
zipdata = zipfl.read(zinfo[0])
zipstring = StringIO(zipdata.decode())
try:
dfp = pd.read_csv(
zipstring,
dtype=indexdb.fno_col_typ,
parse_dates=["TIMESTAMP", "EXPIRY_DT"],
error_bad_lines=False,
)
# Sometimes options type column is named as OPTIONTYPE instead of OPTION_TYP
if "OPTIONTYPE" in dfp.columns:
dfp = dfp.rename(columns={"OPTIONTYPE": "OPTION_TYP"})
return dfp[indexdb.fno_cols]
except Exception as e:
msg = f"Error processing {dt:%d-%b-%Y}"
print_exception(msg)
print_exception(e)
return None
else:
print(f"Failed at second url {dt:%d-%b-%Y}")
print(url)
print(urls)
return None
else:
print(f"File not found for fno {dt:%d-%b-%Y}")
print(url)
return None
def encrypt_file(filename_ori):
'''
Encrypt a file and store the ciphertext into another file.
This function will generate a random key and iv for encryption.
The filename of the output file will be randomly generated.
Return a dict containing:
{original filename, randomly generated filename, key, iv, tag}
Raise IOError when failed to open the file.
'''
# Read file
data = None
try:
with open(filename_ori, "rb") as f:
data = f.read()
except Exception as e:
log.print_exception(e)
log.print_error("IO error", "cannot open file '%s'" % (filename_ori))
raise
# Encrypt using random key and iv
ret = encrypt_rand(data)
# Write ciphertext using random filename
filename_rand = str(uuid.uuid4())+".data"
try:
with open(filename_rand, "wb") as f:
f.write(ret['cipher'])
except Exception as e:
log.print_exception(e)
log.print_error("IO error", "cannot write to file '%s'"
% (filename_rand))
raise
# Store original filename and the generated filename
ret.pop("cipher", None)
ret['filename_ori'] = filename_ori
ret['filename_rand'] = filename_rand
return ret
def update_index_for_dates(dates):
try:
dfn = indexdb.updateIndexData(dates, "NIFTY%2050", "NIFTY")
dfbn = indexdb.updateIndexData(dates, "NIFTY%20BANK", "BANKNIFTY")
if (dfn is not None) and (dfbn is not None):
df = pd.concat([dfn, dfbn])
df = df[indexdb.idx_final_cols]
df.to_hdf(
"indexdb.hdf",
"idx",
mode="a",
append=True,
format="table",
data_columns=True,
)
print(f"Done index")
else:
print("Nothing to update for index...")
except Exception as e:
print_exception(e)
def decrypt_file(filename_ori, filename_rand, key, iv, tag):
'''
Decrypt a file and write the plaintext to filename_ori.
Raise error when failed to open/write file or decryption error.
Any modification to the cipher will cause the decryption to fail.
'''
ret = None
# Read file
try:
with open(filename_rand, "rb") as f:
data = f.read()
except Exception as e:
log.print_exception(e)
log.print_error("IO error", "cannot open file '%s'" % (filename_rand))
raise
# Decrypt
try:
ret = decrypt(key, iv, tag, data)
except Exception as e:
log.print_exception(e)
log.print_error("error", "failed to decrypt message")
raise
# Write to filename_ori
try:
with open(filename_ori, "wb") as f:
data = f.write(ret)
except Exception as e:
log.print_exception(e)
log.print_error("IO error", "cannot write to file '%s'"
% (filename_rand))
raise
def updateVix_upto_Update():
try:
dates = indexdb.get_next_update_start_date("vix")
if dates is not None:
dfn = indexdb.get_vix(dates)
if dfn is not None:
dfn = dfn.rename(columns=indexdb.vix_col_rename)
dfn.to_hdf(
"indexdb.hdf",
"vix",
mode="a",
append=True,
format="table",
data_columns=True,
)
print(f"Done vix")
else:
print("Nothing to update for vix...")
else:
print("Nothing to update for vix...")
except Exception as e:
print_exception(e)
def updateFNOBhavData_for_given_date(d, force_update=False):
dt = None
try:
if isinstance(d, datetime):
dt = datetime.combine(d, datetime.min.time())
elif isinstance(d, str):
dt = parser.parse(d)
except Exception as e:
print(f"Error processing input date {d}")
print_exception(e)
try:
dfd = pd.read_hdf(
"indexdb.hdf",
"fno",
where="SYMBOL==NIFTY & INSTRUMENT==FUTIDX & TIMESTAMP==dt",
columns=["TIMESTAMP"],
)
if len(dfd) == 0:
dfd = None
elif force_update:
try:
store = pd.HDFStore("indexdb.hdf")
store.remove("fno", where="TIMESTAMP==dt")
store.close()
except:
print("Unable to force update.")
dfd = False
except:
dfd = None
if dfd is None:
dfc = indexdb.get_fno_csv_data(dt)
if dfc is not None:
try:
dfc = dfc.reset_index(drop=True).query(
"SYMBOL=='NIFTY' | SYMBOL=='BANKNIFTY'"
)
dfc.to_hdf(
"indexdb.hdf",
"fno",
mode="a",
append=True,
format="table",
data_columns=True,
)
print(f"Done fno {dt:%d%b%Y}")
except Exception as e:
print(f"Error saving data to hdf {dt:%d%b%Y}")
dfc.to_excel(f"{dt:%d-%b-%Y}.xlsx")
print_exception(f"Error in processing {dt:%d-%b-%Y}")
print_exception(e)
else:
print(f"Data already updated for given date {dt:%d-%b-%Y}")
def registrate(username, password):
url = base_url + "registration"
payload = {
'client_id': username,
'client_secret': password
}
try:
r = requests.post(url, data=payload, verify=cert)
except Exception as e:
log.print_exception(e)
log.print_error("error", "connection failure")
return False
# Parse result
try:
response = json.loads(r.text)
except Exception as e:
log.print_exception(e)
log.print_error("authentication failure", "failed to decode server message '%s'" % (r.text))
return False
if response.get("status"):
return True
else:
return False