def main(): parser = argparse.ArgumentParser(description="Retrieves a file from a PyCDM node") parser.add_argument("filename", help="name of file to retrieve") parser.add_argument("-a", "--address", default="127.0.0.1", help="ip address of host") parser.add_argument("-p", "--port", default="8080", help="port of host") args = parser.parse_args() filename = args.filename URL = "http://" + args.address + ":" + args.port + "/" print 'retrieving manifest for {} from {}'.format(filename,URL) resp = requests.get(URL + 'manifest/' + filename) manifest = resp.content fget = FGet() fget.URL = URL rfile = fget.assembleFiles(manifest, 512) f = Filer() f.writeFile(rfile,filename + '.ret')
def get_value(self, key): """ attempts to retrieve a chunk with the specified key value """ #setup zmq connection context = zmq.Context.instance() req = context.socket(zmq.REQ) req.connect(self.PROTOCOL + self.ADDRESS + ':' + self.PORT) print('C: geting chunk...') msg = json.dumps(make_value_request(key)) req.send(msg.encode()) res = req.recv().decode() print('C: chunk retrieved...') js = json.loads(res) data = b64dec(js['body']) #check to make sure there was content if data is not b'': fi = Filer() fi.write_chunk(data, key) print('C: wrote chunk...') else: print('C: chunk has no data...') #terminate connection req.close() context.term()
def __init__(self, config_file=None): self.config_dir = os.path.join( os.path.dirname(os.path.abspath(__file__)), seperator[1], 'environment') self.default_config_file = os.path.join(self.config_dir, 'default', 'env.properties') self.custom_config_file = os.path.join(self.config_dir, 'custom', 'env.properties') self.config = self.read_config_file(self.default_config_file) if not os.path.exists(os.path.dirname(self.custom_config_file)): try: os.makedirs(os.path.dirname(self.custom_config_file)) f = Filer() f.createFile(self.custom_config_file) except Exception as e: print "Error Detected: {0}".format(e) return False elif not os.path.isfile(self.custom_config_file): try: f = Filer() f.createFile(self.custom_config_file) except Exception as e: print "Error Detected: {0}".format(e) return False # SectionName Option DefaultValueIfNotSet self.default_config_dictionary = { 'Authentication': { 'username': '******', 'password': '******' }, 'SMTP': { 'hostname': 'localhost', 'port': '25', 'debug': True, 'mail_domain': 'mydomain' }, 'Notification': { 'author': 'me@mydomain', 'alpha': 'me', 'beta': '', 'release': '' }, }
def test_read_file_from_csv(self): f = Filer() result = f.read_csv("test_renee.csv") expected = [[ 'EMPID', 'GENDER', 'AGE', 'SALES', 'BMI', 'SALARY', 'BIRTHDAY' ], ['A001', 'M', '26', '200', 'Normal', '20', '08-10-1991'], ['A002', 'F', '26', '300', 'Normal', '30', '09-10-1991']] self.assertEqual(expected, result, "didnt read file correctly")
def test_filer_read_excel(self): """ Test if it can read an xlsx document :return: list """ f = Filer() result = f.read_excel("test.xlsx") expected = [['A001', 'M', 26, 200, 'Normal', 20, '08-10-1991'], ['A002', 'F', 26, 300, 'Normal', 30, '09-10-1991']] self.assertEqual(result, expected, "file does not match")
def test_filer_save_excel_and_read_saved(self): """ Test if it can save to excel :return: """ f = Filer() li1 = [['A001', 'M', '26', '200', 'Normal', '20', '08-10-1991'], ['A002', 'F', '26', '300', 'Normal', '30', '09-10-1991']] f.save_excel('doc_test22.xlsx', li1) result = f.read_excel("doc_test22.xlsx") expected = [['A002', 'F', 26, 300, 'Normal', 30, '09-10-1991']] self.assertEqual(result, expected, "saved file couldn't be read")
def upload_files(self): filer = Filer() for p in config.PATHS: # get url to zip file zipfile = filer.archive_file(p) dropfile = '/' + config.DROP_BOX_FOLDER + '/' + path.normpath( path.basename(zipfile)) # upload file to dropbox with open(zipfile, 'rb') as f: self.db.files_upload(f.read(), dropfile, mode=files.WriteMode('overwrite')) # delete file after upload filer.delete_file(zipfile)
def calc(self, event): self.filer = Filer() self.events = self.get_events() self.parser = LineParser() result = '' for event in self.events: channel = self.parser.parse(event) if channel: result += channel.get_info() + '\n' print(channel.get_info()) self.filer.writefile(settings.output_file, result)
def process_file(self, filename): """ attempts to store a file into CSDM """ #attempt to open file if os.path.exists(filename): with open(filename, 'rb') as f: data = f.read() else: print('C: file does not exist...') return print('C: checking for public key...') #make sure public key exists if os.path.exists('public.ecc'): with open('public.ecc', 'rb') as f: epub = f.read() else: print('C: public ecc key does not exist...') print('C: please run: client.py gen-key <passphrase>') return #setup ecc object print('C: encrypting...') epgp = ECCPGP() epgp._epub = epub pack = epgp.encrypt(data, filename) #process the file into an eccpgp pack print('C: processing file...') fi = Filer() manifest = fi.process_data(pack['payload'].encode()) manifest['header'] = pack['header'] #encrypt the manifest manstr = json.dumps(manifest['manifest']) manenc = b64enc(epgp.raw_enc(manstr.encode(), epub)) manifest['manifest'] = manenc #write manifest to disk with open(MANIFESTDIR + manifest['hash'], 'wb') as f: f.write(json.dumps(manifest).encode()) print('C: chunks and manifest created...')
def __init__(self): # create a camera self.camera = DashCam() # create a Filer self.filer = Filer() # create a light manager and turn on the power LED self.lights = LightManager() self.lights.setLED([0], True) # create a button manager self.buttons = ButtonManager() # create constants self.TICK_RATE = 0.125 self.PASSIVE_LEN = 10.0 * 60 # log that a new session has begun self.filer.log( "---------- New Session: " + str(datetime.datetime.now()) + " ----------\n", True)
def assemble_file(self, key, passphrase): """ attempts to decrypt and assemble the local file defined by the passed manifest key """ #attempt to load the manifest print('assembling....') with open(MANIFESTDIR + key, 'rb') as f: manifest = json.loads(f.read().decode()) #create eccpgp object epgp = ECCPGP() epgp.generate(passphrase) #attempt to decrypt manifest manenc = manifest['manifest'] mandec = epgp.raw_dec(b64dec(manenc), passphrase.encode()) manifest['manifest'] = json.loads(mandec.decode()) data = b'' #assemble all the blocks fi = Filer() #size = len(manifest['manifest']) #i = 0 for chunk in manifest['manifest']: data += fi.read_chunk(chunk) #print('{}% complete'.format(int((i/size)*100))) #i += 1 #create eccpgp structure and assign data struct = epgp.make_eccpgp_structure() struct['header'] = manifest['header'] struct['payload'] = data.decode() #decrypt the structure into a pgp package pack = epgp.decrypt(struct, passphrase) #write the decrypted package to assembly directory with open(ASSEMBLEDIR + pack['header']['filename'], 'wb') as f: f.write(pack['payload']) print('file assembled:\t' + ASSEMBLEDIR + pack['header']['filename'])
def __init__(self): # create the filer, light manager, button manager, and dumper self.filer = Filer() self.lights = LightManager() self.buttons = ButtonManager() self.dumper = Dumper("dashdrive") # create constants self.TICK_RATE = 0.125 self.WAIT_TIME = 5.0 # try to log a new config session try: self.filer.log( "---------- New Config Session: " + str(datetime.datetime.now()) + " ----------\n", True) except: # assume the device's storage is full. Wipe it and try again self.wipeFiles(False) self.__init__()
class Controller: f = Filer() v = Validator() db = EmployeeDatabase() chart = ChartMaker() gd = GetData() # Jono's def load_file(self, file): try: if ".csv" in file[-4:]: content = self.f.read_csv(file) elif ".xlsx" in file[-5:]: content = self.f.read_excel(file) elif ".txt" in file[-4:]: content = self.f.read_txt(file) else: message = "incorrect format please see help load" raise NameError(message) validated_employees = self.validate_items(content) return validated_employees except NameError as e: print(e) except FileNotFoundError: print("File not found") except Exception as e: print(e) def validate_items(self, content): valid_employees = [] invalid_employees = [] for employee in content: item = self.v.validate_all(employee) if False in item: invalid_employees.append(employee) else: valid_employees.append(employee) emp = {'Valid': valid_employees, 'Invalid': invalid_employees} return emp def add_to_database(self, content_list): for item in content_list: self.gd.insert_employee(item) def get_all_employees(self): try: employees = self.gd.get_all_employee() if len(employees) > 0: return employees else: raise IndexError except IndexError: print("No employees in database") except Exception as e: print(e) # renee def print_chart_average(self): ave_sales = self.gd.get_ave_sales() ave_salary = self.gd.get_ave_salary() self.chart.make_bar_average(ave_sales, ave_salary) def print_chart_sales(self): employees = self.gd.get_all_employee() self.chart.make_bar_sales(employees) def print_chart_pie(self): employees = self.gd.get_all_employee() self.chart.make_pie(employees) def print_chart_line(self): employees = self.gd.get_all_employee() self.chart.make_line(employees) # Chami def save_file(self, file_format, employee_list): """ # Chami -- added 19-03-2018 >>> Controller.save_file('test1.txt', [['A001', 'M', '26', '200', 'Normal', '20', '08-10-1991'],['A001', 'M', '26', '200', 'Normal', '20', '08-10-1991']]) Traceback (most recent call last): TypeError: save_file() missing 1 required positional argument: 'employee_list' >>> Controller.save_file('test1.txt',) Traceback (most recent call last): TypeError: save_file() missing 2 required positional arguments: 'file_format' and 'employee_list' >>> Controller.save_file() Traceback (most recent call last): TypeError: save_file() missing 3 required positional arguments: 'self', 'file_format', and 'employee_list' >>> Controller.save_file('test1.txt', [[],[]]) Traceback (most recent call last): TypeError: save_file() missing 1 required positional argument: 'employee_list' >>> Controller.save_file('test1.txt', [[,[]]) Traceback (most recent call last): SyntaxError: invalid syntax >>> Controller.save_file('test1.txt', [['A001', 'M', '26', '200', 'Normal', '20', '08-10-1991']['A001', 'M', '26', '200', 'Normal', '20', '08-10-1991']]) Traceback (most recent call last): TypeError: list indices must be integers or slices, not tuple >>> Controller.save_file(' ', [['A001', 'M', '26', '200', 'Normal', '20', '08-10-1991']['A001', 'M', '26', '200', 'Normal', '20', '08-10-1991']]) Traceback (most recent call last): TypeError: list indices must be integers or slices, not tuple >>> Controller.save_file(' .csv', [['A001', 'M', '26', '200', 'Normal', '20', '08-10-1991']['A001', 'M', '26', '200', 'Normal', '20', '08-10-1991']]) Traceback (most recent call last): TypeError: list indices must be integers or slices, not tuple >>> Controller.save_file('test1.', [['A001', 'M', '26', '200', 'Normal', '20', '08-10-1991']['A001', 'M', '26', '200', 'Normal', '20', '08-10-1991']]) Traceback (most recent call last): TypeError: list indices must be integers or slices, not tuple >>> Controller.save_file('.txt', [['A001', 'M', '26', '200', 'Normal', '20', '08-10-1991']['A001', 'M', '26', '200', 'Normal', '20', '08-10-1991']]) Traceback (most recent call last): TypeError: list indices must be integers or slices, not tuple >>> Controller.save_file('.xlsx', [['A001', 'M', '26', '200', 'Normal', '20', '08-10-1991']['A001', 'M', '26', '200', 'Normal', '20', '08-10-1991']]) Traceback (most recent call last): TypeError: list indices must be integers or slices, not tuple >>> Controller.save_file('.', [['A001', 'M', '26', '200', 'Normal', '20', '08-10-1991']['A001', 'M', '26', '200', 'Normal', '20', '08-10-1991']]) Traceback (most recent call last): TypeError: list indices must be integers or slices, not tuple >>> Controller.save_file(' .xlsx', [['A001', 'M', '26', '200', 'Normal', '20', '08-10-1991']['A001', 'M', '26', '200', 'Normal', '20', '08-10-1991']]) Traceback (most recent call last): TypeError: list indices must be integers or slices, not tuple >>> Controller.save_file('.xxx', [['A001', 'M', '26', '200', 'Normal', '20', '08-10-1991']['A001', 'M', '26', '200', 'Normal', '20', '08-10-1991']]) Traceback (most recent call last): TypeError: list indices must be integers or slices, not tuple """ try: if ".csv" in file_format: self.f.save_csv(file_format, employee_list) elif ".xlsx" in file_format: self.f.save_excel(file_format, employee_list) elif ".txt" in file_format: self.f.save_txt_file(file_format, employee_list) else: raise NameError("can not save that file type") except NameError as e: print(e) # except exception part needed def save_invalid(self, invalid_employees): try: self.f.save_txt_file("invalid.csv", invalid_employees) except: print("Whoops something went wrong")
def retrieve(self, key, passphrase): """ attempts to retrieve and re-assemble a file with a given manifest key """ #setup zmq connection context = zmq.Context.instance() req = context.socket(zmq.REQ) req.connect(self.PROTOCOL + self.ADDRESS + ':' + self.PORT) #if manifest doesnt exist locally, request it if not os.path.exists(MANIFESTDIR + key): print('C: requesting manifest for {}'.format(key)) msg = json.dumps(make_manifest_request(key)) req.send(msg.encode()) res = req.recv().decode() js = json.loads(res) #read in the manifest manifest = json.loads(js['body']) #store manifest locally with open(MANIFESTDIR + key, 'wb') as f: f.write(js['body'].encode()) else: with open(MANIFESTDIR + key, 'r') as f: manifest = json.loads(f.read()) print('C: manifest retrieved...') #print(manifest) #setup the eccpgp object epgp = ECCPGP() epgp.generate(passphrase) #decrypt the manifest manenc = manifest['manifest'] mandec = epgp.raw_dec(b64dec(manenc), passphrase.encode()) manifest['manifest'] = json.loads(mandec.decode()) data = b'' #get each chunk described in the manifest print('C: retrieving chunks...') fi = Filer() for chunk in manifest['manifest']: msg = json.dumps(make_value_request(chunk)) req.send(msg.encode()) res = req.recv().decode() js = json.loads(res) data = b64dec(js['body']) fi.write_chunk(data, chunk) #with open(CHUNKDIR+chunk, 'wb') as w: # w.write(data) print('C: chunks saved...') #terminate connection req.close() context.term()
import sys from PySide2.QtWidgets import QApplication from filer import Filer if __name__ == '__main__': # Create the Qt Application app = QApplication(sys.argv) # Create and show the form form = Filer() form.show() # Run the main Qt loop sys.exit(app.exec_())
def test_name_error_save_cvs_exception(self): f = Filer() self.assertRaises(NameError, f.save_csv('#####.csv', [1, 2, 3]))
def __init__(self): self.cf = CSVFactory() self.f = Filer() self.v = Validator() self.db = EmployeeDatabase() self.chart = ChartMaker()
def test_can_not_open_csv_general_exception(self): f = Filer() self.assertRaises(Exception, f.read_csv('test'))
def test_read_not_found_csv_exception(self): f = Filer() with self.assertRaises(FileNotFoundError) as context: f.read_csv('unfound.csv') self.assertTrue('No such file or directory')
def test_can_not_save_csv_general_exception(self): f = Filer() self.assertRaises(Exception, f.save_csv('test', [1, 2, 3]))
def test_can_not_save_csv_file_exist_error_exception(self): f = Filer() ans = f.save_csv('test_renee.csv', [1, 2, 3]) self.assertRaises(FileExistsError, ans)
class Server: cdm = None url = None nickname = None hashname = None fi = Filer() ADDRESS = '127.0.0.1' PORT = '8080' PROTOCOL = 'tcp://' def __init__(self, args): pass def interupt_signal(self, signum, frame): """ process an interrupt signal """ print('\nS: Shutting Down...') self.cleanup() def cleanup(self): """ perform any cleanup """ self.running = False self.router.close() self.context.term() def setup(self): """ perform any needed setup """ signal.signal(signal.SIGINT, self.interupt_signal) self.cache_manifest_names() def cache_manifest_names(self): tups = os.walk('manifests') tup = tups.__next__() self.manifests = tup[2] print(self.manifests) def run(self): """ Run ZMQ server and handle connections """ self.setup() self.context = zmq.Context.instance() self.router = self.context.socket(zmq.ROUTER) #self.router.bind('tcp://127.0.0.1:8080') self.router.bind(self.PROTOCOL + self.ADDRESS + ':' + self.PORT) self.poller = zmq.Poller() self.poller.register(self.router, zmq.POLLIN) self.running = True while self.running: try: #poll and timeout every 60th of a second to do other processing socks = dict(self.poller.poll(timeout=1000)) except Exception as e: if e is zmq.error.Again or zmq.error.ZMQError: self.cleanup() break print('S: something else: ' + str(Exception)) self.cleanup() break if socks.get(self.router) == zmq.POLLIN: id, null, data = self.router.recv_multipart() if data is not None: print("S: received: " + data.decode()) j = json.loads(data.decode()) response = self.process(j) else: response = json.dumps(csdm.make_status_response('BAD')) self.router.send_multipart([id, null, response.encode()]) else: #print('S: doing other...') self.do_other() print('S: Exiting...') def process(self, js): if 'manifest' == js['header']['cmd']: key = js['body'] if not self.fi.manifest_exists(key): #dont have manifest, need to search network for manifest return json.dumps(csdm.make_status_response('NO_FILE')) manifest = self.fi.read_manifest_file(key) response = csdm.make_status_response('OK') response['body'] = manifest return json.dumps(response) elif 'value' == js['header']['cmd']: key = js['body'] if not self.fi.chunk_exists(key): #dont have chunk, need to search network for chunk return json.dumps(csdm.make_status_response('NO_FILE')) chunk = self.fi.read_chunk(key) response = csdm.make_status_response('OK') response['body'] = base64.b64encode(chunk).decode() return json.dumps(response) elif 'manifest-list' == js['header']['cmd']: response = csdm.make_status_response('OK') #print('manifests {}'.format(self.manifests)) response['body'] = self.manifests return json.dumps(response) elif 'ping' == js['header']['cmd']: response = csdm.make_status_response('OK') response['body'] = 'pong' return json.dumps(response) elif 'friend-request' == js['header']['cmd']: response = csdm.make_status_response('OK') common = self.process_friend_reqest(js) response['body'] = { 'status': 'received', 'stage': 0, 'common': common } return json.dumps(response) else: return json.dumps(csdm.make_status_response('OK')) def do_other(self): #print("S: processing other...") pass def process_friend_reqest(self, request): print("S: received friend request...") if request['body']['stage'] == 0: #create new diffie object for this friend and gen the common dif = DiffieHash() common = dif.gen_key() friend = csdm.make_friend_struct() friend['private'] = dif.private friend['common'] = common friend['public'] = dif.make_public(common) return common elif request['body']['stage'] == 1: return None
import sys import requests import argparse import hashlib from cdm import CDMNode, CDM from flask import Flask, request, make_response from filer import Filer #globals app = Flask(__name__) port = None address = None url = None nickname = None hashname = None f = Filer() cdm = CDM() def respond(html, status_code, data=None): """wraps the make_response function""" return make_response(html, status_code, data=data) #return about @app.route('/') def index(): return about()
from docopt import docopt import sys import base64 import json from csdm import make_value_request, make_manifest_request, make_manifest_list_request, make_ping_request, make_friend_request import os from filer import Filer, MANIFESTDIR, CHUNKDIR, ASSEMBLEDIR, check_dirs, CHUNKSIZE from eccpgp import ECCPGP, b64enc, b64dec from diffie import DiffieHash import asyncio import uvloop from paws.paws import get from paws.pahttp import http_data_create fi = Filer() async def do_get_manifest(host, port, key): print(f'C: requesting manifest for {key}') msg = json.dumps(make_manifest_request(key)) data = await get(url=f'http://{host}/manifest/{key}', port=port, body=msg) req = http_data_create(data) print('C: manifest retrieved...') #print('\n{}'.format(req.body)) return req.body async def do_get_manifest_list(host, port): print('C: requesting manifest list...') data = await get(url=f'http://{host}/manifest_list', port=port) req = http_data_create(data) print('C: manifest list retrieved...')
def test_os_error_read_csv_exception(self): f = Filer() self.assertRaises(TypeError, f.read_csv('test2.db'))