def main():
parser = argparse.ArgumentParser(description="Retrieves a file from a PyCDM node")
parser.add_argument("filename", help="name of file to retrieve")
parser.add_argument("-a", "--address", default="127.0.0.1",
help="ip address of host")
parser.add_argument("-p", "--port", default="8080",
help="port of host")
args = parser.parse_args()
filename = args.filename
URL = "http://" + args.address + ":" + args.port + "/"
print 'retrieving manifest for {} from {}'.format(filename,URL)
resp = requests.get(URL + 'manifest/' + filename)
manifest = resp.content
fget = FGet()
fget.URL = URL
rfile = fget.assembleFiles(manifest, 512)
f = Filer()
f.writeFile(rfile,filename + '.ret')
def get_value(self, key):
"""
attempts to retrieve a chunk with the specified key value
"""
#setup zmq connection
context = zmq.Context.instance()
req = context.socket(zmq.REQ)
req.connect(self.PROTOCOL + self.ADDRESS + ':' + self.PORT)
print('C: geting chunk...')
msg = json.dumps(make_value_request(key))
req.send(msg.encode())
res = req.recv().decode()
print('C: chunk retrieved...')
js = json.loads(res)
data = b64dec(js['body'])
#check to make sure there was content
if data is not b'':
fi = Filer()
fi.write_chunk(data, key)
print('C: wrote chunk...')
else:
print('C: chunk has no data...')
#terminate connection
req.close()
context.term()
def __init__(self, config_file=None):
self.config_dir = os.path.join(
os.path.dirname(os.path.abspath(__file__)), seperator[1],
'environment')
self.default_config_file = os.path.join(self.config_dir, 'default',
'env.properties')
self.custom_config_file = os.path.join(self.config_dir, 'custom',
'env.properties')
self.config = self.read_config_file(self.default_config_file)
if not os.path.exists(os.path.dirname(self.custom_config_file)):
try:
os.makedirs(os.path.dirname(self.custom_config_file))
f = Filer()
f.createFile(self.custom_config_file)
except Exception as e:
print "Error Detected: {0}".format(e)
return False
elif not os.path.isfile(self.custom_config_file):
try:
f = Filer()
f.createFile(self.custom_config_file)
except Exception as e:
print "Error Detected: {0}".format(e)
return False
# SectionName Option DefaultValueIfNotSet
self.default_config_dictionary = {
'Authentication': {
'username': '******',
'password': '******'
},
'SMTP': {
'hostname': 'localhost',
'port': '25',
'debug': True,
'mail_domain': 'mydomain'
},
'Notification': {
'author': 'me@mydomain',
'alpha': 'me',
'beta': '',
'release': ''
},
}
def test_read_file_from_csv(self):
f = Filer()
result = f.read_csv("test_renee.csv")
expected = [[
'EMPID', 'GENDER', 'AGE', 'SALES', 'BMI', 'SALARY', 'BIRTHDAY'
], ['A001', 'M', '26', '200', 'Normal', '20', '08-10-1991'],
['A002', 'F', '26', '300', 'Normal', '30', '09-10-1991']]
self.assertEqual(expected, result, "didnt read file correctly")
def test_filer_read_excel(self):
"""
Test if it can read an xlsx document
:return: list
"""
f = Filer()
result = f.read_excel("test.xlsx")
expected = [['A001', 'M', 26, 200, 'Normal', 20, '08-10-1991'],
['A002', 'F', 26, 300, 'Normal', 30, '09-10-1991']]
self.assertEqual(result, expected, "file does not match")
def test_filer_save_excel_and_read_saved(self):
"""
Test if it can save to excel
:return:
"""
f = Filer()
li1 = [['A001', 'M', '26', '200', 'Normal', '20', '08-10-1991'],
['A002', 'F', '26', '300', 'Normal', '30', '09-10-1991']]
f.save_excel('doc_test22.xlsx', li1)
result = f.read_excel("doc_test22.xlsx")
expected = [['A002', 'F', 26, 300, 'Normal', 30, '09-10-1991']]
self.assertEqual(result, expected, "saved file couldn't be read")
def upload_files(self):
filer = Filer()
for p in config.PATHS:
# get url to zip file
zipfile = filer.archive_file(p)
dropfile = '/' + config.DROP_BOX_FOLDER + '/' + path.normpath(
path.basename(zipfile))
# upload file to dropbox
with open(zipfile, 'rb') as f:
self.db.files_upload(f.read(),
dropfile,
mode=files.WriteMode('overwrite'))
# delete file after upload
filer.delete_file(zipfile)
def calc(self, event):
self.filer = Filer()
self.events = self.get_events()
self.parser = LineParser()
result = ''
for event in self.events:
channel = self.parser.parse(event)
if channel:
result += channel.get_info() + '\n'
print(channel.get_info())
self.filer.writefile(settings.output_file, result)
def process_file(self, filename):
"""
attempts to store a file into CSDM
"""
#attempt to open file
if os.path.exists(filename):
with open(filename, 'rb') as f:
data = f.read()
else:
print('C: file does not exist...')
return
print('C: checking for public key...')
#make sure public key exists
if os.path.exists('public.ecc'):
with open('public.ecc', 'rb') as f:
epub = f.read()
else:
print('C: public ecc key does not exist...')
print('C: please run: client.py gen-key <passphrase>')
return
#setup ecc object
print('C: encrypting...')
epgp = ECCPGP()
epgp._epub = epub
pack = epgp.encrypt(data, filename)
#process the file into an eccpgp pack
print('C: processing file...')
fi = Filer()
manifest = fi.process_data(pack['payload'].encode())
manifest['header'] = pack['header']
#encrypt the manifest
manstr = json.dumps(manifest['manifest'])
manenc = b64enc(epgp.raw_enc(manstr.encode(), epub))
manifest['manifest'] = manenc
#write manifest to disk
with open(MANIFESTDIR + manifest['hash'], 'wb') as f:
f.write(json.dumps(manifest).encode())
print('C: chunks and manifest created...')
def __init__(self):
# create a camera
self.camera = DashCam()
# create a Filer
self.filer = Filer()
# create a light manager and turn on the power LED
self.lights = LightManager()
self.lights.setLED([0], True)
# create a button manager
self.buttons = ButtonManager()
# create constants
self.TICK_RATE = 0.125
self.PASSIVE_LEN = 10.0 * 60
# log that a new session has begun
self.filer.log(
"---------- New Session: " + str(datetime.datetime.now()) +
" ----------\n", True)
def assemble_file(self, key, passphrase):
"""
attempts to decrypt and assemble the local file defined by the passed manifest key
"""
#attempt to load the manifest
print('assembling....')
with open(MANIFESTDIR + key, 'rb') as f:
manifest = json.loads(f.read().decode())
#create eccpgp object
epgp = ECCPGP()
epgp.generate(passphrase)
#attempt to decrypt manifest
manenc = manifest['manifest']
mandec = epgp.raw_dec(b64dec(manenc), passphrase.encode())
manifest['manifest'] = json.loads(mandec.decode())
data = b''
#assemble all the blocks
fi = Filer()
#size = len(manifest['manifest'])
#i = 0
for chunk in manifest['manifest']:
data += fi.read_chunk(chunk)
#print('{}% complete'.format(int((i/size)*100)))
#i += 1
#create eccpgp structure and assign data
struct = epgp.make_eccpgp_structure()
struct['header'] = manifest['header']
struct['payload'] = data.decode()
#decrypt the structure into a pgp package
pack = epgp.decrypt(struct, passphrase)
#write the decrypted package to assembly directory
with open(ASSEMBLEDIR + pack['header']['filename'], 'wb') as f:
f.write(pack['payload'])
print('file assembled:\t' + ASSEMBLEDIR + pack['header']['filename'])
def __init__(self):
# create the filer, light manager, button manager, and dumper
self.filer = Filer()
self.lights = LightManager()
self.buttons = ButtonManager()
self.dumper = Dumper("dashdrive")
# create constants
self.TICK_RATE = 0.125
self.WAIT_TIME = 5.0
# try to log a new config session
try:
self.filer.log(
"---------- New Config Session: " +
str(datetime.datetime.now()) + " ----------\n", True)
except:
# assume the device's storage is full. Wipe it and try again
self.wipeFiles(False)
self.__init__()
class Controller:
f = Filer()
v = Validator()
db = EmployeeDatabase()
chart = ChartMaker()
gd = GetData()
# Jono's
def load_file(self, file):
try:
if ".csv" in file[-4:]:
content = self.f.read_csv(file)
elif ".xlsx" in file[-5:]:
content = self.f.read_excel(file)
elif ".txt" in file[-4:]:
content = self.f.read_txt(file)
else:
message = "incorrect format please see help load"
raise NameError(message)
validated_employees = self.validate_items(content)
return validated_employees
except NameError as e:
print(e)
except FileNotFoundError:
print("File not found")
except Exception as e:
print(e)
def validate_items(self, content):
valid_employees = []
invalid_employees = []
for employee in content:
item = self.v.validate_all(employee)
if False in item:
invalid_employees.append(employee)
else:
valid_employees.append(employee)
emp = {'Valid': valid_employees, 'Invalid': invalid_employees}
return emp
def add_to_database(self, content_list):
for item in content_list:
self.gd.insert_employee(item)
def get_all_employees(self):
try:
employees = self.gd.get_all_employee()
if len(employees) > 0:
return employees
else:
raise IndexError
except IndexError:
print("No employees in database")
except Exception as e:
print(e)
# renee
def print_chart_average(self):
ave_sales = self.gd.get_ave_sales()
ave_salary = self.gd.get_ave_salary()
self.chart.make_bar_average(ave_sales, ave_salary)
def print_chart_sales(self):
employees = self.gd.get_all_employee()
self.chart.make_bar_sales(employees)
def print_chart_pie(self):
employees = self.gd.get_all_employee()
self.chart.make_pie(employees)
def print_chart_line(self):
employees = self.gd.get_all_employee()
self.chart.make_line(employees)
# Chami
def save_file(self, file_format, employee_list):
"""
# Chami -- added 19-03-2018
>>> Controller.save_file('test1.txt', [['A001', 'M', '26', '200', 'Normal', '20', '08-10-1991'],['A001', 'M', '26', '200', 'Normal', '20', '08-10-1991']])
Traceback (most recent call last):
TypeError: save_file() missing 1 required positional argument: 'employee_list'
>>> Controller.save_file('test1.txt',)
Traceback (most recent call last):
TypeError: save_file() missing 2 required positional arguments: 'file_format' and 'employee_list'
>>> Controller.save_file()
Traceback (most recent call last):
TypeError: save_file() missing 3 required positional arguments: 'self', 'file_format', and 'employee_list'
>>> Controller.save_file('test1.txt', [[],[]])
Traceback (most recent call last):
TypeError: save_file() missing 1 required positional argument: 'employee_list'
>>> Controller.save_file('test1.txt', [[,[]])
Traceback (most recent call last):
SyntaxError: invalid syntax
>>> Controller.save_file('test1.txt', [['A001', 'M', '26', '200', 'Normal', '20', '08-10-1991']['A001', 'M', '26', '200', 'Normal', '20', '08-10-1991']])
Traceback (most recent call last):
TypeError: list indices must be integers or slices, not tuple
>>> Controller.save_file(' ', [['A001', 'M', '26', '200', 'Normal', '20', '08-10-1991']['A001', 'M', '26', '200', 'Normal', '20', '08-10-1991']])
Traceback (most recent call last):
TypeError: list indices must be integers or slices, not tuple
>>> Controller.save_file(' .csv', [['A001', 'M', '26', '200', 'Normal', '20', '08-10-1991']['A001', 'M', '26', '200', 'Normal', '20', '08-10-1991']])
Traceback (most recent call last):
TypeError: list indices must be integers or slices, not tuple
>>> Controller.save_file('test1.', [['A001', 'M', '26', '200', 'Normal', '20', '08-10-1991']['A001', 'M', '26', '200', 'Normal', '20', '08-10-1991']])
Traceback (most recent call last):
TypeError: list indices must be integers or slices, not tuple
>>> Controller.save_file('.txt', [['A001', 'M', '26', '200', 'Normal', '20', '08-10-1991']['A001', 'M', '26', '200', 'Normal', '20', '08-10-1991']])
Traceback (most recent call last):
TypeError: list indices must be integers or slices, not tuple
>>> Controller.save_file('.xlsx', [['A001', 'M', '26', '200', 'Normal', '20', '08-10-1991']['A001', 'M', '26', '200', 'Normal', '20', '08-10-1991']])
Traceback (most recent call last):
TypeError: list indices must be integers or slices, not tuple
>>> Controller.save_file('.', [['A001', 'M', '26', '200', 'Normal', '20', '08-10-1991']['A001', 'M', '26', '200', 'Normal', '20', '08-10-1991']])
Traceback (most recent call last):
TypeError: list indices must be integers or slices, not tuple
>>> Controller.save_file(' .xlsx', [['A001', 'M', '26', '200', 'Normal', '20', '08-10-1991']['A001', 'M', '26', '200', 'Normal', '20', '08-10-1991']])
Traceback (most recent call last):
TypeError: list indices must be integers or slices, not tuple
>>> Controller.save_file('.xxx', [['A001', 'M', '26', '200', 'Normal', '20', '08-10-1991']['A001', 'M', '26', '200', 'Normal', '20', '08-10-1991']])
Traceback (most recent call last):
TypeError: list indices must be integers or slices, not tuple
"""
try:
if ".csv" in file_format:
self.f.save_csv(file_format, employee_list)
elif ".xlsx" in file_format:
self.f.save_excel(file_format, employee_list)
elif ".txt" in file_format:
self.f.save_txt_file(file_format, employee_list)
else:
raise NameError("can not save that file type")
except NameError as e:
print(e)
# except exception part needed
def save_invalid(self, invalid_employees):
try:
self.f.save_txt_file("invalid.csv", invalid_employees)
except:
print("Whoops something went wrong")
def retrieve(self, key, passphrase):
"""
attempts to retrieve and re-assemble a file with a given manifest key
"""
#setup zmq connection
context = zmq.Context.instance()
req = context.socket(zmq.REQ)
req.connect(self.PROTOCOL + self.ADDRESS + ':' + self.PORT)
#if manifest doesnt exist locally, request it
if not os.path.exists(MANIFESTDIR + key):
print('C: requesting manifest for {}'.format(key))
msg = json.dumps(make_manifest_request(key))
req.send(msg.encode())
res = req.recv().decode()
js = json.loads(res)
#read in the manifest
manifest = json.loads(js['body'])
#store manifest locally
with open(MANIFESTDIR + key, 'wb') as f:
f.write(js['body'].encode())
else:
with open(MANIFESTDIR + key, 'r') as f:
manifest = json.loads(f.read())
print('C: manifest retrieved...')
#print(manifest)
#setup the eccpgp object
epgp = ECCPGP()
epgp.generate(passphrase)
#decrypt the manifest
manenc = manifest['manifest']
mandec = epgp.raw_dec(b64dec(manenc), passphrase.encode())
manifest['manifest'] = json.loads(mandec.decode())
data = b''
#get each chunk described in the manifest
print('C: retrieving chunks...')
fi = Filer()
for chunk in manifest['manifest']:
msg = json.dumps(make_value_request(chunk))
req.send(msg.encode())
res = req.recv().decode()
js = json.loads(res)
data = b64dec(js['body'])
fi.write_chunk(data, chunk)
#with open(CHUNKDIR+chunk, 'wb') as w:
# w.write(data)
print('C: chunks saved...')
#terminate connection
req.close()
context.term()
import sys
from PySide2.QtWidgets import QApplication
from filer import Filer
if __name__ == '__main__':
# Create the Qt Application
app = QApplication(sys.argv)
# Create and show the form
form = Filer()
form.show()
# Run the main Qt loop
sys.exit(app.exec_())
def test_name_error_save_cvs_exception(self):
f = Filer()
self.assertRaises(NameError, f.save_csv('#####.csv', [1, 2, 3]))
def __init__(self):
self.cf = CSVFactory()
self.f = Filer()
self.v = Validator()
self.db = EmployeeDatabase()
self.chart = ChartMaker()
def test_can_not_open_csv_general_exception(self):
f = Filer()
self.assertRaises(Exception, f.read_csv('test'))
def test_read_not_found_csv_exception(self):
f = Filer()
with self.assertRaises(FileNotFoundError) as context:
f.read_csv('unfound.csv')
self.assertTrue('No such file or directory')
def test_can_not_save_csv_general_exception(self):
f = Filer()
self.assertRaises(Exception, f.save_csv('test', [1, 2, 3]))
def test_can_not_save_csv_file_exist_error_exception(self):
f = Filer()
ans = f.save_csv('test_renee.csv', [1, 2, 3])
self.assertRaises(FileExistsError, ans)
class Server:
cdm = None
url = None
nickname = None
hashname = None
fi = Filer()
ADDRESS = '127.0.0.1'
PORT = '8080'
PROTOCOL = 'tcp://'
def __init__(self, args):
pass
def interupt_signal(self, signum, frame):
"""
process an interrupt signal
"""
print('\nS: Shutting Down...')
self.cleanup()
def cleanup(self):
"""
perform any cleanup
"""
self.running = False
self.router.close()
self.context.term()
def setup(self):
"""
perform any needed setup
"""
signal.signal(signal.SIGINT, self.interupt_signal)
self.cache_manifest_names()
def cache_manifest_names(self):
tups = os.walk('manifests')
tup = tups.__next__()
self.manifests = tup[2]
print(self.manifests)
def run(self):
"""
Run ZMQ server and handle connections
"""
self.setup()
self.context = zmq.Context.instance()
self.router = self.context.socket(zmq.ROUTER)
#self.router.bind('tcp://127.0.0.1:8080')
self.router.bind(self.PROTOCOL + self.ADDRESS + ':' + self.PORT)
self.poller = zmq.Poller()
self.poller.register(self.router, zmq.POLLIN)
self.running = True
while self.running:
try:
#poll and timeout every 60th of a second to do other processing
socks = dict(self.poller.poll(timeout=1000))
except Exception as e:
if e is zmq.error.Again or zmq.error.ZMQError:
self.cleanup()
break
print('S: something else: ' + str(Exception))
self.cleanup()
break
if socks.get(self.router) == zmq.POLLIN:
id, null, data = self.router.recv_multipart()
if data is not None:
print("S: received: " + data.decode())
j = json.loads(data.decode())
response = self.process(j)
else:
response = json.dumps(csdm.make_status_response('BAD'))
self.router.send_multipart([id, null, response.encode()])
else:
#print('S: doing other...')
self.do_other()
print('S: Exiting...')
def process(self, js):
if 'manifest' == js['header']['cmd']:
key = js['body']
if not self.fi.manifest_exists(key):
#dont have manifest, need to search network for manifest
return json.dumps(csdm.make_status_response('NO_FILE'))
manifest = self.fi.read_manifest_file(key)
response = csdm.make_status_response('OK')
response['body'] = manifest
return json.dumps(response)
elif 'value' == js['header']['cmd']:
key = js['body']
if not self.fi.chunk_exists(key):
#dont have chunk, need to search network for chunk
return json.dumps(csdm.make_status_response('NO_FILE'))
chunk = self.fi.read_chunk(key)
response = csdm.make_status_response('OK')
response['body'] = base64.b64encode(chunk).decode()
return json.dumps(response)
elif 'manifest-list' == js['header']['cmd']:
response = csdm.make_status_response('OK')
#print('manifests {}'.format(self.manifests))
response['body'] = self.manifests
return json.dumps(response)
elif 'ping' == js['header']['cmd']:
response = csdm.make_status_response('OK')
response['body'] = 'pong'
return json.dumps(response)
elif 'friend-request' == js['header']['cmd']:
response = csdm.make_status_response('OK')
common = self.process_friend_reqest(js)
response['body'] = {
'status': 'received',
'stage': 0,
'common': common
}
return json.dumps(response)
else:
return json.dumps(csdm.make_status_response('OK'))
def do_other(self):
#print("S: processing other...")
pass
def process_friend_reqest(self, request):
print("S: received friend request...")
if request['body']['stage'] == 0:
#create new diffie object for this friend and gen the common
dif = DiffieHash()
common = dif.gen_key()
friend = csdm.make_friend_struct()
friend['private'] = dif.private
friend['common'] = common
friend['public'] = dif.make_public(common)
return common
elif request['body']['stage'] == 1:
return None
import sys
import requests
import argparse
import hashlib
from cdm import CDMNode, CDM
from flask import Flask, request, make_response
from filer import Filer
#globals
app = Flask(__name__)
port = None
address = None
url = None
nickname = None
hashname = None
f = Filer()
cdm = CDM()
def respond(html, status_code, data=None):
"""wraps the make_response function"""
return make_response(html, status_code, data=data)
#return about
@app.route('/')
def index():
return about()
from docopt import docopt
import sys
import base64
import json
from csdm import make_value_request, make_manifest_request, make_manifest_list_request, make_ping_request, make_friend_request
import os
from filer import Filer, MANIFESTDIR, CHUNKDIR, ASSEMBLEDIR, check_dirs, CHUNKSIZE
from eccpgp import ECCPGP, b64enc, b64dec
from diffie import DiffieHash
import asyncio
import uvloop
from paws.paws import get
from paws.pahttp import http_data_create
fi = Filer()
async def do_get_manifest(host, port, key):
print(f'C: requesting manifest for {key}')
msg = json.dumps(make_manifest_request(key))
data = await get(url=f'http://{host}/manifest/{key}', port=port, body=msg)
req = http_data_create(data)
print('C: manifest retrieved...')
#print('\n{}'.format(req.body))
return req.body
async def do_get_manifest_list(host, port):
print('C: requesting manifest list...')
data = await get(url=f'http://{host}/manifest_list', port=port)
req = http_data_create(data)
print('C: manifest list retrieved...')
def test_os_error_read_csv_exception(self):
f = Filer()
self.assertRaises(TypeError, f.read_csv('test2.db'))
