def deleteConnection(self, connection): # Delete referenced properties 1st db.delete( GaeConnectionModel.last_active.get_value_for_datastore( connection.model)) db.delete( GaeConnectionModel.connected.get_value_for_datastore( connection.model)) db.delete( GaeConnectionModel.last_polled.get_value_for_datastore( connection.model)) # Optional referenced properties authentication_key = GaeConnectionModel.authentication.get_value_for_datastore( connection.model) if authentication_key is not None: db.delete(authentication_key) session_key = GaeConnectionModel.session.get_value_for_datastore( connection.model) if session_key is not None: db.delete(session_key) # Delete connection connection.model.delete() connection.model = None ConnectionManager.deleteConnection(self, connection) db.run_in_transaction(self._decrementChannelCount, connection.channel_name)
def __init__(self, connection_class=Connection, connection_params=None, mc_servers=['127.0.0.1:11211'], mc_debug=0): ConnectionManager.__init__(self, connection_class=connection_class, connection_params=connection_params) self.mc = self.createMcClient(mc_servers, mc_debug) self._lock = memcache_manager.MemcacheMutex(self.mc)
class UpdateH5iManager: def __init__(self, connectionStr): self.connMgr = ConnectionManager() self.conn = self.connMgr.connectToDatabase(connectionStr) self.cur = self.conn.cursor() def updateDatasetMarkerH5Index(self, datasetId, markerId, idx): print ("Updating %s:%s:%s ..." % (datasetId, markerId, idx)) updateStmt = "update dataset_marker set marker_idx=%s where dataset_id=%s and marker_id=%s;" self.cur.execute(updateStmt, (idx, datasetId, markerId)) def updateDatasetDnarunH5Index(self, datasetId, sampleId, idx): print ("Updating %s:%s:%s ..." % (datasetId, sampleId, idx)) updateStmt = "update dataset_dnarun set dnarun_idx=%s where dataset_id=%s and dnarun_id=%s;" self.cur.execute(updateStmt, (idx, datasetId, sampleId)) def updateSerialSequence(self, tableName, primaryKeyColumnName): updateSeqSql = "SELECT pg_catalog.setval(pg_get_serial_sequence('"+tableName+"', '"+primaryKeyColumnName+"'), MAX("+primaryKeyColumnName+")) FROM "+tableName+";" #print("updateSeqSql = "+updateSeqSql) self.cur.execute(updateSeqSql) def commitTransaction(self): self.conn.commit() def rollbackTransaction(self): self.conn.rollback() def closeConnection(self): self.connMgr.disconnectFromDatabase()
def deleteConnection(self, connection): d = self.connections.delete().\ where(self.connections.c.id==connection.id) db = self.getDb() db.execute(d) db.close() ConnectionManager.deleteConnection(self, connection)
def notice_slaves_stop(self): task_connection = ConnectionManager(queue_name=self.task_queue_name, durable=True, no_ack=False) screen_list = [key for key in self.slave_manager.slave_dict.keys()] for screen in screen_list: task_connection.publish('STOP {}') # task_connection.broadcast_task('STOP {}') task_connection.stop()
def __init__(self, engine, metadata, connection_class=Connection, connection_params=None, table_prefix=''): ConnectionManager.__init__(self, connection_class=connection_class, connection_params=connection_params) self.engine = engine self.metadata = metadata self.table_prefix = table_prefix and "%s_" % table_prefix.rstrip('_') or table_prefix self.mapTables()
def __init__(self, connection_class=GaeConnection, connection_params=None, touch_time=10000): ConnectionManager.__init__(self, connection_class=connection_class, connection_params=connection_params) # Reduces number of writes to 'last_active' field. self.touch_time = touch_time
def __init__(self): LOG.debug("JellyfinClient initializing...") self.config = Config() self.http = HTTP(self) self.wsc = WSClient(self) self.auth = ConnectionManager(self) self.jellyfin = api.API(self.http) self.callback_ws = callback self.callback = callback
def __get__(self, instance, owner): class_name = _class_name(owner) config_name = owner.config_name config_name = config_name if config_name else class_name config_col = ConnectionManager.get_config(config_name).col if not config_col: collection_name = owner.collection_name document_name = collection_name if collection_name else class_name return ConnectionManager.get_config(config_name).db[document_name] return config_col
def __init__(self, main_window, port, service_id): super().__init__() self.main_window = main_window #ConnectionManager (create Slots) self.connection_manager = ConnectionManager(port, service_id) self.message_manager = MessageManager(self.connection_manager, service_id) self.message_manager.signal_message_received.connect(self.print_string) self.message_manager.signal_election_responded.connect( self.set_election) self.message_manager.signal_leader_responded.connect( self.leader_has_been_found) self.message_manager.signal_has_message_to_send.connect( self.send_udp_message) self.message_manager.signal_leader_alive.connect(self.set_alive) data = {"port": str(port), "leader": "Null"} jdata = json.dumps(data) self.main_window.set_details(jdata) self.main_window.signal_has_message_to_send.connect( self.send_udp_message) self.main_window.signal_has_multicast_to_send.connect( self.send_multicast_message) self.main_window.signal_start_checking.connect(self.close) self.own_port = port self.service_id = service_id self.leader_ip = None self.leader_port = 0 self.leader_id = " " self.election = True self.leader_alive = False self.iam_leader = False self.control_boolean = False self.alive_checker_thread = AliveChecker(self, self.leader_ip, self.leader_port, self.connection_manager) self.alive_checker_thread.signal_election_is_due.connect( self.do_election) self.message_manager.signal_received_ok.connect(self.receive_ok) self.message_manager.signal_critical_request.connect( self.receive_request) self.worker_controller = WorkerController(1) self.worker_controller.signal_request_critical.connect( self.request_critical) self.worker_controller.signal_send_free_message.connect( self.send_free_message) #do first multicast self.th = th.Thread(target=self.request_group) self.th.start()
def __init__(self, port): super(AWSIOTDeviceAdapter, self).__init__() self.set_config('default_timeout', 5.0) reg = ComponentRegistry() endpoint = reg.get_config('awsiot-endpoint') rootcert = reg.get_config('awsiot-rootcert') iamuser = reg.get_config('awsiot-iamkey') iamsecret = reg.get_config('awsiot-iamtoken') iamsession = reg.get_config('awsiot-session', default=None) args = {} args['endpoint'] = endpoint args['root_certificate'] = rootcert args['use_websockets'] = True args['iam_key'] = iamuser args['iam_secret'] = iamsecret args['iam_session'] = iamsession self._logger = logging.getLogger(__name__) # Port should be a topic prefix that allows us to connect # only to subset of IOTile devices managed by a gateway # rather than to directly accessible iotile devices. if port is None: port = "" if len(port) > 0 and port[-1] != '/': port = port + '/' self.client = OrderedAWSIOTClient(args) self.name = str(uuid.uuid4()) self.client.connect(self.name) self.prefix = port self.conns = ConnectionManager(self.id) self.conns.start() self.client.subscribe(self.prefix + 'devices/+/data/advertisement', self._on_advertisement, ordered=False) self._deferred = queue.Queue() self.set_config('minimum_scan_time', 5.0) self.set_config('probe_supported', True) self.set_config('probe_required', True) self.mtu = self.get_config( 'mtu', 60 * 1024) # Split script payloads larger than this
class RCScreenApp(App): """ Responsible for the whole client's application. """ screen_image_format = StringProperty(DEFAULT_SCREEN_IMAGE_FORMAT) username = StringProperty("") password = StringProperty("") # TODO: can connect_screen handle this? is_controller = BooleanProperty(True) # TODO: can connect_screen handle this? partner = StringProperty("") connection_manager = ObjectProperty(ConnectionManager()) x_sensitivity = NumericProperty(10, min=0) y_sensitivity = NumericProperty(10, min=0) screen_size = ListProperty() other_screen_width = NumericProperty(0) other_screen_height = NumericProperty(0) def on_stop(self): """ Close connection_manager """ if self.connection_manager.running: self.connection_manager.close() def build(self): """ when the app is created """ try: self.icon = ICON_PATH except Exception as e: print(e) return super().build()
def setUp(self): httpretty.enable() create_environment() make_fake_dir() with open(CONFIG_FILEPATH, 'r') as fo: self.cfg = json.load(fo) self.auth = (self.cfg['user'], self.cfg['pass']) self.base_url = ''.join( [self.cfg['server_address'], self.cfg['api_suffix']]) self.files_url = ''.join([self.base_url, 'files/']) self.actions_url = ''.join([self.base_url, 'actions/']) self.shares_url = ''.join([self.base_url, 'shares/']) self.user_url = ''.join([self.base_url, 'users/']) self.cm = ConnectionManager(self.cfg)
def __init__(self): self.logger = logging.getLogger('backdoor') self.logger.setLevel(logging.DEBUG) ch = logging.StreamHandler() ch.setLevel(logging.DEBUG) formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') ch.setFormatter(formatter) self.logger.addHandler(ch) self.logger.info('Starting backdoor.') signal.signal(signal.SIGINT, self.stop) self.running = True self.connection_manager = ConnectionManager(config.api_host, config.api_port) self.connection_manager.start() self.methods = {} self.services = [] self.load_services()
class LoadIfileManager: def __init__(self, connectionStr): self.connMgr = ConnectionManager() self.conn = self.connMgr.connectToDatabase(connectionStr) self.cur = self.conn.cursor() self.fdm = ForeignDataManager() #print("Load IFile Manager Initialized.") def dropForeignTable(self, fdwTableName): self.cur.execute("drop foreign table if exists "+fdwTableName+";") #print("drop foreign table if exists "+fdwTableName+";") def createForeignTable(self, iFile, fTableName): header, fdwScript = self.fdm.generateFDWScript(iFile, fTableName) self.cur.execute(fdwScript) return header def createFileWithDerivedIds(self, outputFilePath, derivedIdSql): copyStmt = "copy ("+derivedIdSql+") to '"+outputFilePath+"' with delimiter E'\\t'"+" csv header;" #print("copyStmt = "+copyStmt) self.cur.execute(copyStmt) def createFileWithoutDuplicates(self, outputFilePath, noDupsSql): copyStmt = "copy ("+noDupsSql+") to '"+outputFilePath+"' with delimiter E'\\t'"+" csv header;" #print("copyStmt = "+copyStmt) self.cur.execute(copyStmt) def loadData(self, tableName, header, fileToLoad, primaryKeyColumnName): loadSql = "copy "+tableName+" ("+(",".join(header))+")"+" from '"+fileToLoad+"' with delimiter E'\\t' csv header;" #print("loadSql = "+loadSql) self.updateSerialSequence(tableName, primaryKeyColumnName) self.cur.execute(loadSql) def updateSerialSequence(self, tableName, primaryKeyColumnName): updateSeqSql = "SELECT pg_catalog.setval(pg_get_serial_sequence('"+tableName+"', '"+primaryKeyColumnName+"'), MAX("+primaryKeyColumnName+")) FROM "+tableName+";" #print("updateSeqSql = "+updateSeqSql) self.cur.execute(updateSeqSql) def commitTransaction(self): self.conn.commit() def rollbackTransaction(self): self.conn.rollback() def closeConnection(self): self.connMgr.disconnectFromDatabase()
def __init__(self, my_port, node_host=None, node_port=None): self.my_ip = self.__get_myip() print('Server IP address is set to ... ', self.my_ip) self.my_port = my_port self.cm = ConnectionManager(self.my_ip, self.my_port, self.__handle_message) self.node_host = node_host self.node_port = node_port self.bm = BlockchainManager() self.tp = TransactionPool() self.clock = 0 self.status = STATUS_IDLE self.next_status = STATUS_IDLE self.new_txs = [] self.end_mining_clock = None self.new_block = None self.to_port = None
class JellyfinClient(object): logged_in = False def __init__(self): LOG.debug("JellyfinClient initializing...") self.config = Config() self.http = HTTP(self) self.wsc = WSClient(self) self.auth = ConnectionManager(self) self.jellyfin = api.API(self.http) self.callback_ws = callback self.callback = callback def set_credentials(self, credentials=None): self.auth.credentials.set_credentials(credentials or {}) def get_credentials(self): return self.auth.credentials.get_credentials() def authenticate(self, credentials=None, options=None): self.set_credentials(credentials or {}) state = self.auth.connect(options or {}) if state['State'] == CONNECTION_STATE['SignedIn']: LOG.info("User is authenticated.") self.logged_in = True self.callback("ServerOnline", {'Id': self.auth.server_id}) state['Credentials'] = self.get_credentials() return state def start(self, websocket=False, keep_alive=True): if not self.logged_in: raise ValueError("User is not authenticated.") self.http.start_session() if keep_alive: self.http.keep_alive = True if websocket: self.start_wsc() def start_wsc(self): self.wsc.start() def stop(self): self.wsc.stop_client() self.http.stop_session()
def deleteConnection(self, connection): lock_name = self.getLockName(self.CONNECTIONS_ATTR) self._lock.acquire(lock_name) try: connection_ids = self.mc.get(self.CONNECTIONS_ATTR) for i, connection_id in enumerate(connection_ids): if connection_id == connection.id: connection_ids.pop(i) break self.mc.set(self.CONNECTIONS_ATTR, connection_ids) finally: self._lock.release(lock_name) keys = [self.getKeyName(connection.id, attr) for attr in self.ATTRIBUTES] self.mc.delete_multi(keys) self.decrementChannelCount(connection.channel_name) ConnectionManager.deleteConnection(self, connection)
def deleteConnection(self, connection): # Delete referenced properties 1st db.delete(GaeConnectionModel.last_active.get_value_for_datastore(connection.model)) db.delete(GaeConnectionModel.connected.get_value_for_datastore(connection.model)) db.delete(GaeConnectionModel.last_polled.get_value_for_datastore(connection.model)) # Optional referenced properties authentication_key = GaeConnectionModel.authentication.get_value_for_datastore(connection.model) if authentication_key is not None: db.delete(authentication_key) session_key = GaeConnectionModel.session.get_value_for_datastore(connection.model) if session_key is not None: db.delete(session_key) # Delete connection connection.model.delete() connection.model = None ConnectionManager.deleteConnection(self, connection) db.run_in_transaction(self._decrementChannelCount, connection.channel_name)
def Initialize(port, functionMap={}, asyncHandler=None): ''' Initializes BlackBoard with the corresponding parameters. :param int port: The port through which BlackBoard will communicate with this module. :param dictionary functionMap: A dictionary containing **key:value** pairs, where the *key* is the name of a command received (a string), and the *value* is either a tuple containing a function as a first element and a boolean as a second element, or a function. The function in both cases is the function that is going to execute the specified command and receives on object of type :class:`Command` (See :ref:`Creating a command handler <creating_a_command_handler>`). The boolean value indicates whether the execution of that command should be synchronous (on the same thread) or asynchronous, usually synchronous execution is preferred for fast commands that can answer almost immediately and asynchronous for commands that might take a little time. When the value is only a function, by default the execution is synchronous. *functionMap* can also contain an entry with a string containing only an asterisk, meaning that would be the handler in case no other handler is found for a specific command. .. note:: Notice that although functionMap can include a wildcard handler and this might seem like the module could answer anything, BlackBoard will only send commands that are registered under this module's configuration. :param function asyncHandler: A function that would handle the response of commands when sent with the method :func:`Send` instead of using :func:`SendAndWait`. This means the execution of a program that sends a command could continue and an asynchronous handler would handle the response when one is received. .. note:: Notice that the asyncHandler functionality could also be achieved using a :class:`ParallelSender` object, but it has other implications. ''' global _executors, _connMan, _parser, _p, _initialized, _ready _executors = { 'busy': (lambda x: Response('busy'), False), 'ready': (_isReady, False), 'alive': (lambda x: Response('alive', True), False) } for m in functionMap: if isinstance(functionMap[m], types.FunctionType): _executors[m] = (functionMap[m], False) elif isinstance(functionMap[m], tuple): _executors[m] = functionMap[m] else: print 'Element in function map is not a function nor a correct tuple: ' + repr( functionMap[m]) _connMan = ConnectionManager(port) _parser = CommandParser(asyncHandler) _p = threading.Thread(target=_MainThread) _p.daemon = True _initialized = True
def __init__(self): RegexMatchingEventHandler.__init__(self, ignore_regexes=Daemon.IGNORED_REGEX, ignore_directories=True) # Just Initialize variable the Daemon.start() do the other things self.daemon_state = 'down' # TODO implement the daemon state (disconnected, connected, syncronizing, ready...) self.running = 0 self.client_snapshot = {} # EXAMPLE {'<filepath1>: ['<timestamp>', '<md5>', '<filepath2>: ...} self.local_dir_state = {} # EXAMPLE {'last_timestamp': '<timestamp>', 'global_md5': '<md5>'} self.listener_socket = None self.observer = None self.cfg = self.load_cfg(Daemon.CONFIG_FILEPATH) self.init_sharing_path() self.conn_mng = ConnectionManager(self.cfg)
def __init__(self): self.logger = logging.getLogger('backdoor') self.logger.setLevel(logging.DEBUG) ch = logging.StreamHandler() ch.setLevel(logging.DEBUG) formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') ch.setFormatter(formatter) self.logger.addHandler(ch) self.logger.info('Starting backdoor.') signal.signal(signal.SIGINT, self.stop) self.running = True self.connection_manager = ConnectionManager(config.api_host, config.api_port) self.connection_manager.start()
def __init__(self, cfg_path=None, sharing_path=None): FileSystemEventHandler.__init__(self) # Just Initialize variable the Daemon.start() do the other things self.daemon_state = 'down' # TODO implement the daemon state (disconnected, connected, syncronizing, ready...) self.running = 0 self.client_snapshot = {} # EXAMPLE {'<filepath1>: ['<timestamp>', '<md5>', '<filepath2>: ...} self.local_dir_state = {} # EXAMPLE {'last_timestamp': '<timestamp>', 'global_md5': '<md5>'} self.listener_socket = None self.observer = None self.cfg = self._load_cfg(cfg_path, sharing_path) self.password = self._load_pass() self._init_sharing_path(sharing_path) self.conn_mng = ConnectionManager(self.cfg)
def setUp(self): httpretty.enable() create_environment() make_fake_dir() with open(CONFIG_FILEPATH, 'r') as fo: self.cfg = json.load(fo) self.auth = (self.cfg['user'], self.cfg['pass']) self.base_url = ''.join([self.cfg['server_address'], self.cfg['api_suffix']]) self.files_url = ''.join([self.base_url, 'files/']) self.actions_url = ''.join([self.base_url, 'actions/']) self.shares_url = ''.join([self.base_url, 'shares/']) self.user_url = ''.join([self.base_url, 'users/']) self.cm = ConnectionManager(self.cfg)
def main(): config = ConfigParser() config.read("config.ini") cm_config = { "host": config.get("SQL", "host_address"), "port": config.getint("SQL", "port"), "user": "", "passwd": "", "database": config.get("SQL", "database"), } cm = ConnectionManager(cm_config) ui = UserInterface(cm) try: ui.start() except Exception as e: print(e)
def __init__(self, conf_file): self.config = ConfigParser.ConfigParser(allow_no_value=True) self.clean_time_gap = None self.wait_time_for_slave = None self.master_queue_name = None self.task_queue_name = None self.task_queue_size_limit = None self.task_file_name = None self.task_counter_file = None self.ssh_key = None self.s3_bucket = None self.s3_folder = None self.slave_num_every_packup = None self.slave_max_sec_each_task = None self.slave_python_version = None self.master_ip = None self.slaves_ip = None self.slave_awake_frequency = None self.configure(conf_file) self.last_wake_time = None self.repeated_timer = None self.is_started = False self.pop_forever_handler = None logging.info('Starting task manager...') self.task_manager = TaskManager(self.task_file_name, self.task_counter_file) logging.info('Starting slave manager...') self.slave_manager = SlaveManager(master_ip=self.master_ip, slaves_ip=self.slaves_ip, ssh_key=self.ssh_key, s3_bucket=self.s3_bucket, s3_folder=self.s3_folder, slave_num_every_packup=self.slave_num_every_packup, slave_max_sec_each_task=self.slave_max_sec_each_task, slave_python_version=self.slave_python_version, slave_awake_frequency=self.slave_awake_frequency, slave_buffer_size=1) logging.info('Starting connection manager...') self.message_connection = ConnectionManager(queue_name=self.master_queue_name, durable=False, callback=self.msg_callback, no_ack=True)
def main(): main_app = FastAPI() manager = ConnectionManager() @main_app.websocket("/ws/{client_id}") async def websocket_endpoint(websocket: WebSocket, client_id: UUID4): await manager.connect(websocket, client_id) try: while True: ws_data = await websocket.receive_text() message = Message(sender_id=client_id, **json.loads(ws_data)) print(f'User {message.sender_id} sends to user {message.receiver_id} the message {message.text}') await manager.send_personal_message(message) await manager.send_message_to_target(message) except WebSocketDisconnect: manager.disconnect(websocket, client_id) return main_app
def setUp(self): httpretty.enable() with open(TestConnectionManager.CONFIG_FILEPATH, 'r') as fo: self.cfg = json.load(fo) self.auth = (self.cfg['user'], self.cfg['pass']) # override self.cfg['server_address'] = "http://www.pyboxtest.com" self.cfg['sharing_path'] = os.path.join(os.getcwd(), "sharing_folder") # create this auth testing self.authServerAddress = "http://" + self.cfg['user'] + ":" + self.cfg['pass'] + "@www.pyboxtest.com" # example of self.base_url = 'http://localhost:5000/API/V1/' self.base_url = ''.join([self.cfg['server_address'], self.cfg['api_suffix']]) self.files_url = ''.join([self.base_url, 'files/']) self.actions_url = ''.join([self.base_url, 'actions/']) self.shares_url = ''.join([self.base_url, 'shares/']) self.cm = ConnectionManager(self.cfg) self.make_fake_dir()
def start_popping_tasks(self): task_connection = ConnectionManager(queue_name=self.task_queue_name, durable=True, no_ack=False) eof_reached = False while self.is_started and not eof_reached: current_task_queue_size = task_connection.get_task_queue_size() while self.is_started and current_task_queue_size < self.task_queue_size_limit: task = self.task_manager.pop_task() if task is None: # TODO: Don't use Error. Just break and handle the case later in this function logging.info('EOF Reached') eof_reached = True break message = 'WORK ' + ujson.dumps(task) task_connection.publish(message) current_task_queue_size += 1 task_connection.stop()
def __get__(self, instance, owner): config_name = owner.config_name config_name = config_name if config_name else _class_name(owner) return getattr(ConnectionManager.get_config(config_name), self.attr_name)
class TestConnectionManager(unittest.TestCase): CONFIG_DIR = os.path.join(os.environ['HOME'], '.PyBox') CONFIG_FILEPATH = os.path.join(CONFIG_DIR, 'daemon_config') LOCAL_DIR_STATE_PATH = os.path.join(CONFIG_DIR, 'dir_state') def setUp(self): httpretty.enable() with open(TestConnectionManager.CONFIG_FILEPATH, 'r') as fo: self.cfg = json.load(fo) self.auth = (self.cfg['user'], self.cfg['pass']) # override self.cfg['server_address'] = "http://www.pyboxtest.com" self.cfg['sharing_path'] = os.path.join(os.getcwd(), "sharing_folder") # create this auth testing self.authServerAddress = "http://" + self.cfg['user'] + ":" + self.cfg['pass'] + "@www.pyboxtest.com" # example of self.base_url = 'http://localhost:5000/API/V1/' self.base_url = ''.join([self.cfg['server_address'], self.cfg['api_suffix']]) self.files_url = ''.join([self.base_url, 'files/']) self.actions_url = ''.join([self.base_url, 'actions/']) self.shares_url = ''.join([self.base_url, 'shares/']) self.cm = ConnectionManager(self.cfg) self.make_fake_dir() # files: @httpretty.activate def test_download_normal_file(self): url = ''.join((self.files_url, 'file.txt')) httpretty.register_uri(httpretty.GET, url, status=201) data = {'filepath': 'file.txt'} response = self.cm.do_download(data) self.assertEqual(response, True) @httpretty.activate def test_download_file_not_exists(self): url = ''.join((self.files_url, 'file.tx')) httpretty.register_uri(httpretty.GET, url, status=404) data = {'filepath': 'file.tx'} response = self.cm.do_download(data) self.assertEqual(response, False) @httpretty.activate def test_do_upload_success(self): # prepare fake server url = ''.join((self.files_url, 'foo.txt')) js = json.dumps({"server_timestamp": time.time()}) recv_js = json.loads(js) httpretty.register_uri(httpretty.POST, url, status=201, body=js, content_type="application/json") # call api response = self.cm.do_upload({'filepath': 'foo.txt'}) self.assertEqual(response, recv_js) # actions: @httpretty.activate def test_do_move(self): url = ''.join((self.actions_url, 'move')) js = json.dumps({"server_timestamp": time.time()}) recv_js = json.loads(js) httpretty.register_uri(httpretty.POST, url, status=201, body=js, content_type="application/json") response = self.cm.do_move({'src': 'foo.txt', 'dst': 'folder/foo.txt'}) self.assertEqual(response, recv_js) @httpretty.activate def test_do_delete(self): url = ''.join((self.actions_url, 'delete')) js = json.dumps({"server_timestamp": time.time()}) recv_js = json.loads(js) httpretty.register_uri(httpretty.POST, url, status=201, body=js, content_type="application/json") d = {'filepath': 'foo.txt'} response = self.cm.do_delete(d) self.assertEqual(response, recv_js) @httpretty.activate def test_do_modify(self): url = ''.join((self.files_url, 'foo.txt')) js = json.dumps({"server_timestamp": time.time()}) recv_js = json.loads(js) httpretty.register_uri(httpretty.PUT, url, status=201, body=js, content_type="application/json") response = self.cm.do_modify({'filepath': 'foo.txt'}) self.assertEqual(response, recv_js) @httpretty.activate def test_do_copy(self): url = ''.join([self.actions_url, 'copy']) d = {'src': 'foo.txt', 'dst': 'folder/foo.txt'} js = json.dumps({"server_timestamp": time.time()}) recv_js = json.loads(js) httpretty.register_uri(httpretty.POST, url, status=201, body=js, content_type="application/json") response = self.cm.do_copy(d) self.assertEqual(response, recv_js) @httpretty.activate def test_get_server_snapshot(self): url = self.files_url js = json.dumps({'files': 'foo.txt'}) httpretty.register_uri(httpretty.GET, url, status=201, body=js, content_type="application/json") response = self.cm.do_get_server_snapshot('') self.assertEqual(json.dumps(response), js) def tearDown(self): httpretty.disable() httpretty.reset() self.remove_fake_dir() def make_fake_dir(self): sharing_path = os.path.join(self.cfg['sharing_path']) if os.path.exists(sharing_path): shutil.rmtree(sharing_path) else: os.makedirs(os.path.join(self.cfg['sharing_path'])) fake_file = os.path.join(self.cfg['sharing_path'], 'foo.txt') with open(fake_file, 'w') as fc: fc.write('foo.txt :)') def remove_fake_dir(self): shutil.rmtree(os.path.join(self.cfg['sharing_path']))
class DataLogger: """ In this class the different systems are initialiased: - logger and its handlers - local configuration - connection with internet If there is no connection with the internet: - A timed job is created that controls internet connection. - Logging of data is started with local configuration. - If the internet connection is started, and the online\ configuration differs, the old wrongly logged data will be removed. If there is connection with the internet and the server is working: - Check if the online configuration differs from the local one. If so,\ the configuration will be updated. - Logging of data is started. - Sending of data is started. - A timed job is created that checks if the online configuration is\ updated - The management of leds is started. """ def __init__(self): try: # initiate logger self.logger = logging.getLogger() self.logger.setLevel(logging.DEBUG) self.log_send_store_handler = LogSendStoreHandler(LOG_LOCATION) formatter = logging.Formatter( '%(asctime)s - %(levelname)s - %(name)s - %(message)s') self.log_send_store_handler.setFormatter(formatter) self.logger.addHandler(self.log_send_store_handler) self.logger.info('Initialising system...') job_info_filter = JobInfoFilter() logging.getLogger('apscheduler.scheduler').addFilter( job_info_filter) logging.getLogger('apscheduler.threadpool').addFilter( job_info_filter) # load local configuration self.conf_man = ConfigurationManager(CONFIG_LOCATION) self.log_send_store_handler.update_configuration() self.scheduler = Scheduler() self.scheduler.start() self.packet_manager = PacketManager(self.scheduler) # initiate network connection self.connection = ConnectionManager() # add scheduler and connection to log handler self.log_send_store_handler.update_configuration( scheduler=self.scheduler, connection=self.connection) # try to connect connected_to_internet = self.connection.check_internet_connection() connected_to_server = self.connection.check_server_connection() if connected_to_internet and connected_to_server: self.load_online_configuration_and_initiate_sending_data() self.packet_manager.update_time() self.packet_manager.initiate_send_packets(self.connection) else: ''' if there is no connection: keep checking for a connection temporarily use offline timer and modbus slave configuration ''' if connected_to_internet: self.packet_manager.update_time() self.wait_for_connection_to_load_configuration() # initiate sensor timers self.read_sensor_scheduler = ReadSensorScheduler( self.scheduler, self.packet_manager) self.led_manager = LedManager(self.scheduler) self.led_manager.update_led(PinName.powered, LedState.on) self.set_up_led_manager_calls() # sleep 2 seconds to intialise led of log handler sleep(1) self.logger.info('Initialisation complete') while True: sleep(10) self.logger.debug('Alive and kicking') if self.logger.level is logging.DEBUG: scheduler_jobs = self.scheduler.get_jobs() if len(scheduler_jobs) > 1: self.logger.debug('Current scheduler jobs:') for index, job in enumerate(scheduler_jobs): self.logger.debug(' Job {0}: {1} {2}'.format( index, job.name, job.next_run_time)) else: self.logger.debug('No running scheduler jobs') except Exception as e: self.logger.error(e) raise self.log_send_store_handler.send_logs_job() def load_online_configuration_and_initiate_sending_data(self): # check online configuration try: online_checksum = self.connection.get_configuration_checksum() self.logger.info("Checking online configuration..") if self.conf_man.is_online_configuration_different(online_checksum): self.logger.info( 'Online configuration is new, updating configuration..') online_configuration = self.connection.get_configuration() self.conf_man.validate_json_configuration(online_configuration) self.conf_man.save_configuration_local( online_checksum, online_configuration) self.packet_manager.remove_all_packets_from_memory() # update systems that make use of the configuration self.log_send_store_handler.update_configuration( scheduler=self.scheduler, connection=self.connection) self.connection.update_configuration() try: self.read_sensor_scheduler.update_configuration() except: pass self.packet_manager.update_configuration() except: self.logger.warning('Problem updating configuration') raise try: # try to remove job self.scheduler.unschedule_func( self.load_online_configuration_and_initiate_sending_data) except: pass # periodically check changes in configuration self.scheduler.add_interval_job( self.load_online_configuration_and_initiate_sending_data, seconds=configuration.get_time_interval_to_check_online_config()) self.packet_manager.initiate_send_packets(self.connection) def wait_for_connection_to_load_configuration(self): if not self.connection.is_connected(): # no internet connection, start job to check connection self.scheduler.add_interval_job(self.try_to_connect_to_internet, seconds=CHECK_CONNECTION_INTERVAL) else: self.packet_manager.update_time() if not self.connection.check_server_connection(): # no connection with server, start job to check connection self.scheduler.add_interval_job( self.try_to_load_online_configuration, seconds=CHECK_CONNECTION_INTERVAL) def try_to_connect_to_internet(self): if self.connection.check_internet_connection(): self.scheduler.unschedule_func(self.try_to_connect_to_internet) self.packet_manager.update_time() if not self.connection.check_server_connection(): # no connection with server, start job to check connection self.scheduler.add_interval_job( self.try_to_load_online_configuration, seconds=CHECK_CONNECTION_INTERVAL) else: self.load_online_configuration_and_initiate_sending_data() def try_to_load_online_configuration(self): if self.connection.check_server_connection(): self.load_online_configuration_and_initiate_sending_data() self.scheduler.unschedule_func( self.try_to_load_online_configuration) def set_up_led_manager_calls(self): sensor_led_call = LedCall(self.led_manager, PinName.readingsensor) connected_led_call = LedCall(self.led_manager, PinName.connected) logging_led_call = LedCall(self.led_manager, PinName.logging) self.read_sensor_scheduler.set_led_call(sensor_led_call) self.connection.set_led_call(connected_led_call) self.log_send_store_handler.set_led_call(logging_led_call)
class TestConnectionManager(unittest.TestCase): def setUp(self): httpretty.enable() create_environment() with open(CONFIG_FILEPATH, 'r') as fo: self.cfg = json.load(fo) self.auth = (self.cfg['user'], self.cfg['pass']) # override self.cfg['server_address'] = "http://www.pyboxtest.com" self.cfg['sharing_path'] = os.path.join(os.getcwd(), "sharing_folder") # create this auth testing self.authServerAddress = "http://" + self.cfg['user'] + ":" + self.cfg['pass'] + "@www.pyboxtest.com" # example of self.base_url = 'http://localhost:5000/API/V1/' self.base_url = ''.join([self.cfg['server_address'], self.cfg['api_suffix']]) self.files_url = ''.join([self.base_url, 'files/']) self.actions_url = ''.join([self.base_url, 'actions/']) self.shares_url = ''.join([self.base_url, 'shares/']) self.user_url = ''.join([self.base_url, 'users/']) self.cm = ConnectionManager(self.cfg) self.make_fake_dir() @httpretty.activate def test_register_user(self): """ Test register user api: method = POST resource = <user> data = password=<password> """ data = (USR, PW) url = ''.join((self.user_url, USR)) content = 'user activated' content_jsoned = json.dumps(content) httpretty.register_uri(httpretty.POST, url, status=200, body=content_jsoned) response = self.cm.do_register(data) self.assertIn('content', response) self.assertEqual(response['content'], content) @httpretty.activate def test_register_user_with_weak_password(self): """ Test register user api with weak password: method = POST resource = <user> data = password=<password> """ weak_password = '******' data = (USR, weak_password) url = ''.join((self.user_url, USR)) content = {'type_of_improvement': 'improvement suggested'} content_jsoned = json.dumps(content) httpretty.register_uri(httpretty.POST, url, status=403, body=content_jsoned) response = self.cm.do_register(data) self.assertIn('improvements', response) self.assertEqual(response['improvements'], content) @httpretty.activate def test_register_user_with_already_existent_user(self): """ Test register user api with already existent user: method = POST resource = <user> data = password=<password> """ data = (USR, PW) url = ''.join((self.user_url, USR)) # This is the only case where server doesn't send data with the message error httpretty.register_uri(httpretty.POST, url, status=409) response = self.cm.do_register(data) self.assertIn('content', response) self.assertIsInstance(response['content'], str) @httpretty.activate def test_fail_to_register_user(self): """ Test failed register request Test activate user api: method = POST resource = <user> data = password=<password> """ data = (USR, PW) url = ''.join((self.user_url, USR)) httpretty.register_uri(httpretty.POST, url, status=500) response = self.cm.do_register(data) self.assertIsInstance(response['content'], str) self.assertFalse(response['successful']) @httpretty.activate def test_activate_user(self): """ Test successful activation Test activate user api: method = PUT resource = <user> data = activation_code=<token> """ user = '******' token = '6c9fb345c317ad1d31ab9d6445d1a820' data = (user, token) url = ''.join((self.user_url, user)) answer = 'user activated' answer_jsoned = json.dumps(answer) httpretty.register_uri(httpretty.PUT, url, status=201, body=answer_jsoned) response = self.cm.do_activate(data) self.assertIsInstance(response['content'], unicode) self.assertTrue(response['successful']) @httpretty.activate def test_activate_user_already_existent(self): """ Test activate user already existent Test activate user api: method = PUT resource = <user> data = activation_code=<token> """ user = '******' token = 'bad_token' data = (user, token) url = ''.join((self.user_url, user)) httpretty.register_uri(httpretty.PUT, url, status=409) response = self.cm.do_activate(data) self.assertIsInstance(response['content'], str) self.assertFalse(response['successful']) @httpretty.activate def test_activate_user_not_existent(self): """ Test activate user not existent Test activate user api: method = PUT resource = <user> data = activation_code=<token> """ user = '******' token = 'bad_token' data = (user, token) url = ''.join((self.user_url, user)) httpretty.register_uri(httpretty.PUT, url, status=404) response = self.cm.do_activate(data) self.assertIsInstance(response['content'], str) self.assertFalse(response['successful']) @httpretty.activate def test_fail_to_activate_user(self): """ Test failed activation request Test activate user api: method = PUT resource = <user> data = activation_code=<token> """ user = '******' token = 'bad_token' data = (user, token) url = ''.join((self.user_url, user)) httpretty.register_uri(httpretty.PUT, url, status=500) response = self.cm.do_activate(data) self.assertIsInstance(response['content'], str) self.assertFalse(response['successful']) @httpretty.activate def test_post_recover_password_not_found(self): """ Test that if /users/<email>/reset POST == 404 then cm return None """ # An unknown user (neither registered nor pending) is a resource not found for the server... email = '*****@*****.**' url = self.user_url + email + '/reset' # ...so the server should return a 404: httpretty.register_uri(httpretty.POST, url, status=404) # and the command manager must return None in this case response = self.cm.do_reqrecoverpass(email) self.assertIsNone(response) @httpretty.activate def test_post_recover_password_accept(self): """ Test that if /users/<email>/reset POST == 202 then cm return True """ email = '*****@*****.**' url = self.user_url + email + '/reset' httpretty.register_uri(httpretty.POST, url, status=202) response = self.cm.do_reqrecoverpass(email) self.assertTrue(response) @httpretty.activate def test_put_recover_password_not_found(self): """ Test that if /users/<email> PUT == 404 then cm return None """ email = '*****@*****.**' recoverpass_code = os.urandom(16).encode('hex') new_password = '******' url = self.user_url + email httpretty.register_uri(httpretty.PUT, url, status=404) data = email, recoverpass_code, new_password response = self.cm.do_recoverpass(data) self.assertFalse(response) @httpretty.activate def test_put_recover_password_ok(self): """ Test that if /users/<email> PUT == 200 then cm return True """ email = '*****@*****.**' recoverpass_code = os.urandom(16).encode('hex') new_password = '******' url = self.user_url + email httpretty.register_uri(httpretty.PUT, url, status=200) data = email, recoverpass_code, new_password response = self.cm.do_recoverpass(data) self.assertTrue(response) # files: @httpretty.activate def test_download_normal_file(self): url = ''.join((self.files_url, 'file.txt')) httpretty.register_uri(httpretty.GET, url, status=201) data = {'filepath': 'file.txt'} response = self.cm.do_download(data) self.assertEqual(response, True) @httpretty.activate def test_download_file_not_exists(self): url = ''.join((self.files_url, 'file.tx')) httpretty.register_uri(httpretty.GET, url, status=404) data = {'filepath': 'file.tx'} response = self.cm.do_download(data) self.assertEqual(response, False) @httpretty.activate def test_do_upload_success(self): # prepare fake server url = ''.join((self.files_url, 'foo.txt')) js = json.dumps({"server_timestamp": time.time()}) recv_js = json.loads(js) httpretty.register_uri(httpretty.POST, url, status=201, body=js, content_type="application/json") # call api response = self.cm.do_upload({'filepath': 'foo.txt', 'md5': 'test_md5'}) self.assertEqual(response, recv_js) # actions: @httpretty.activate def test_do_move(self): url = ''.join((self.actions_url, 'move')) js = json.dumps({"server_timestamp": time.time()}) recv_js = json.loads(js) httpretty.register_uri(httpretty.POST, url, status=201, body=js, content_type="application/json") response = self.cm.do_move({'src': 'foo.txt', 'dst': 'folder/foo.txt'}) self.assertEqual(response, recv_js) @httpretty.activate def test_do_delete(self): url = ''.join((self.actions_url, 'delete')) js = json.dumps({"server_timestamp": time.time()}) recv_js = json.loads(js) httpretty.register_uri(httpretty.POST, url, status=201, body=js, content_type="application/json") d = {'filepath': 'foo.txt'} response = self.cm.do_delete(d) self.assertEqual(response, recv_js) @httpretty.activate def test_do_modify(self): url = ''.join((self.files_url, 'foo.txt')) js = json.dumps({"server_timestamp": time.time()}) recv_js = json.loads(js) httpretty.register_uri(httpretty.PUT, url, status=201, body=js, content_type="application/json") response = self.cm.do_modify({'filepath': 'foo.txt', 'md5': 'test_md5'}) self.assertEqual(response, recv_js) @httpretty.activate def test_do_copy(self): url = ''.join([self.actions_url, 'copy']) d = {'src': 'foo.txt', 'dst': 'folder/foo.txt'} js = json.dumps({"server_timestamp": time.time()}) recv_js = json.loads(js) httpretty.register_uri(httpretty.POST, url, status=201, body=js, content_type="application/json") response = self.cm.do_copy(d) self.assertEqual(response, recv_js) @httpretty.activate def test_get_server_snapshot(self): url = self.files_url js = json.dumps({'files': 'foo.txt'}) httpretty.register_uri(httpretty.GET, url, status=201, body=js, content_type="application/json") response = self.cm.do_get_server_snapshot('') self.assertEqual(json.dumps(response), js) def tearDown(self): httpretty.disable() httpretty.reset() self.remove_fake_dir() def make_fake_dir(self): sharing_path = os.path.join(self.cfg['sharing_path']) if os.path.exists(sharing_path): shutil.rmtree(sharing_path) else: os.makedirs(os.path.join(self.cfg['sharing_path'])) fake_file = os.path.join(self.cfg['sharing_path'], 'foo.txt') with open(fake_file, 'w') as fc: fc.write('foo.txt :)') def remove_fake_dir(self): shutil.rmtree(os.path.join(self.cfg['sharing_path']))
class TestConnectionManager(unittest.TestCase): CONFIG_DIR = os.path.join(os.environ['HOME'], '.PyBox') CONFIG_FILEPATH = os.path.join(CONFIG_DIR, 'daemon_config') LOCAL_DIR_STATE_PATH = os.path.join(CONFIG_DIR, 'dir_state') def setUp(self): httpretty.enable() with open(TestConnectionManager.CONFIG_FILEPATH, 'r') as fo: self.cfg = json.load(fo) self.auth = (self.cfg['user'], self.cfg['pass']) # override self.cfg['server_address'] = "http://www.pyboxtest.com" self.cfg['sharing_path'] = os.path.join(os.getcwd(), "sharing_folder") # create this auth testing self.authServerAddress = "http://" + self.cfg['user'] + ":" + self.cfg['pass'] + "@www.pyboxtest.com" # example of self.base_url = 'http://localhost:5000/API/V1/' self.base_url = ''.join([self.cfg['server_address'], self.cfg['api_suffix']]) self.files_url = ''.join([self.base_url, 'files/']) self.actions_url = ''.join([self.base_url, 'actions/']) self.shares_url = ''.join([self.base_url, 'shares/']) self.user_url = ''.join([self.base_url, 'users/']) self.cm = ConnectionManager(self.cfg) self.make_fake_dir() @httpretty.activate def test_register_user(self): """ Test register user api: method = POST resource = <user> data = password=<password> """ data = (USR, PW) url = ''.join((self.user_url, USR)) content = 'user activated' content_jsoned = json.dumps(content) httpretty.register_uri(httpretty.POST, url, status=200, body= content_jsoned) response = self.cm.do_register(data) self.assertIn('content', response) self.assertEqual(response['content'], content) @httpretty.activate def test_register_user_with_weak_password(self): """ Test register user api with weak password: method = POST resource = <user> data = password=<password> """ weak_password = '******' data = (USR, weak_password) url = ''.join((self.user_url, USR)) content = {'type_of_improvement': 'improvement suggested'} content_jsoned = json.dumps(content) httpretty.register_uri(httpretty.POST, url, status=403, body=content_jsoned) response = self.cm.do_register(data) self.assertIn('improvements', response) self.assertEqual(response['improvements'], content) @httpretty.activate def test_register_user_with_already_existent_user(self): """ Test register user api with already existent user: method = POST resource = <user> data = password=<password> """ data = (USR, PW) url = ''.join((self.user_url, USR)) # This is the only case where server doesn't send data with the message error httpretty.register_uri(httpretty.POST, url, status=409) response = self.cm.do_register(data) response = self.cm.do_register(data) self.assertIn('content', response) self.assertIsInstance(response['content'], str) @httpretty.activate def test_activate_user(self): """ Test activate user api: method = PUT resource = <user> data = activation_code=<token> """ user = '******' token = '6c9fb345c317ad1d31ab9d6445d1a820' data = (user, token) url = ''.join((self.user_url, user)) httpretty.register_uri(httpretty.PUT, url, status=201, body='user activated') response = self.cm.do_activate(data) self.assertNotEqual(response, False) self.assertIsInstance(response, unicode) httpretty.register_uri(httpretty.PUT, url, status=404) self.assertFalse(self.cm.do_activate(data)) httpretty.register_uri(httpretty.PUT, url, status=409) self.assertFalse(self.cm.do_activate(data)) # files: @httpretty.activate def test_download_normal_file(self): url = ''.join((self.files_url, 'file.txt')) httpretty.register_uri(httpretty.GET, url, status=201) data = {'filepath': 'file.txt'} response = self.cm.do_download(data) self.assertEqual(response, True) @httpretty.activate def test_download_file_not_exists(self): url = ''.join((self.files_url, 'file.tx')) httpretty.register_uri(httpretty.GET, url, status=404) data = {'filepath': 'file.tx'} response = self.cm.do_download(data) self.assertEqual(response, False) @httpretty.activate def test_do_upload_success(self): # prepare fake server url = ''.join((self.files_url, 'foo.txt')) js = json.dumps({"server_timestamp": time.time()}) recv_js = json.loads(js) httpretty.register_uri(httpretty.POST, url, status=201, body=js, content_type="application/json") # call api response = self.cm.do_upload({'filepath': 'foo.txt', 'md5': 'test_md5'}) self.assertEqual(response, recv_js) # actions: @httpretty.activate def test_do_move(self): url = ''.join((self.actions_url, 'move')) js = json.dumps({"server_timestamp": time.time()}) recv_js = json.loads(js) httpretty.register_uri(httpretty.POST, url, status=201, body=js, content_type="application/json") response = self.cm.do_move({'src': 'foo.txt', 'dst': 'folder/foo.txt'}) self.assertEqual(response, recv_js) @httpretty.activate def test_do_delete(self): url = ''.join((self.actions_url, 'delete')) js = json.dumps({"server_timestamp": time.time()}) recv_js = json.loads(js) httpretty.register_uri(httpretty.POST, url, status=201, body=js, content_type="application/json") d = {'filepath': 'foo.txt'} response = self.cm.do_delete(d) self.assertEqual(response, recv_js) @httpretty.activate def test_do_modify(self): url = ''.join((self.files_url, 'foo.txt')) js = json.dumps({"server_timestamp": time.time()}) recv_js = json.loads(js) httpretty.register_uri(httpretty.PUT, url, status=201, body=js, content_type="application/json") response = self.cm.do_modify({'filepath': 'foo.txt', 'md5': 'test_md5'}) self.assertEqual(response, recv_js) @httpretty.activate def test_do_copy(self): url = ''.join([self.actions_url, 'copy']) d = {'src': 'foo.txt', 'dst': 'folder/foo.txt'} js = json.dumps({"server_timestamp": time.time()}) recv_js = json.loads(js) httpretty.register_uri(httpretty.POST, url, status=201, body=js, content_type="application/json") response = self.cm.do_copy(d) self.assertEqual(response, recv_js) @httpretty.activate def test_get_server_snapshot(self): url = self.files_url js = json.dumps({'files': 'foo.txt'}) httpretty.register_uri(httpretty.GET, url, status=201, body=js, content_type="application/json") response = self.cm.do_get_server_snapshot('') self.assertEqual(json.dumps(response), js) def tearDown(self): httpretty.disable() httpretty.reset() self.remove_fake_dir() def make_fake_dir(self): sharing_path = os.path.join(self.cfg['sharing_path']) if os.path.exists(sharing_path): shutil.rmtree(sharing_path) else: os.makedirs(os.path.join(self.cfg['sharing_path'])) fake_file = os.path.join(self.cfg['sharing_path'], 'foo.txt') with open(fake_file, 'w') as fc: fc.write('foo.txt :)') def remove_fake_dir(self): shutil.rmtree(os.path.join(self.cfg['sharing_path']))
def sahibinden_scraper(num_page): """ Web scraper for sahibinden.com param num_page: number of page that wanted to be scraped """ cm = ConnectionManager() start_time = time.time() data_ids = [] seller_names = [] seller_links = [] ad_titles = [] ad_links = [] prices = [] counties = [] districts = [] other_items = [ ] # this is a container list for attirbutes like date, area, room, floor within building, etc. diff = num_page - 2 last = ((diff * 20) + 20) + 1 for t in range(0, last, 20): url = 'https://www.sahibinden.com/kiralik-daire/antalya?pagingOffset=' + str( t) r = cm.request(url) status_code = r.code if r != '' else -1 # Status code of request. if status_code == 200: so = bs4.BeautifulSoup(r.read(), 'lxml') table = so.find('table', id='searchResultsTable') tracker = 0 requests = 0 for i in table.findAll('tr'): if i.get( 'data-id' ) is None: # means it is not a home ad. Maybe just a google ad. continue # ignore these data points else: # if data points are home ads ids = i.get('data-id') names_seller = [ _.get('title') for _ in (i.find_all( 'a', attrs={'class': 'titleIcon store-icon'})) ] links_seller = [ _.get('href') for _ in (i.find_all( 'a', attrs={'class': 'titleIcon store-icon'})) ] titles_ad = [ ' '.join(_.text.split()) for _ in i.find_all('a', attrs={'class': 'classifiedTitle'}) ] links_ad = [ 'https://www.sahibinden.com' + _.get('href') for _ in i.find_all('a', attrs={'class': 'classifiedTitle'}) ] data_ids.append(ids) seller_names.append(names_seller) seller_links.append(links_seller) ad_titles.append(titles_ad) ad_links.append(links_ad) #finding prices corresponding to ads for z in i.find_all( 'td', attrs={'class': 'searchResultsPriceValue'}): price = ' '.join(z.text.split()) if '.' in price: # if price is like 7.100 # convert it to 7100 and keep it in 'prices' list prices.append(price.replace('.', '')) else: # if it is not, just put it to the 'prices' list prices.append(price) # finding addresses corresponding to ads for f in i.findAll( 'td', attrs={'class': 'searchResultsLocationValue'}): turkish_adress_name = ''.join(f.text.split( )) # name of county and district in Turkish #converting Turkish charecters to English charecters normalized = unicodedata.normalize( 'NFD', turkish_adress_name) #converted_adress_name consists of county and district names. converted_adress_name = u"".join([ c for c in normalized if not unicodedata.combining(c) ]) #first_letters is a list consists of first letters of county and district names. For example: ['M', 'F'] first_letters = re.findall('[A-Z]+', converted_adress_name) # if district name is not specified, then 'first_letters' will contain only one letter, like ['A'] try: # first_letters[1] is a district name # first_letters[0] is a county name districts.append(converted_adress_name[ converted_adress_name. find(first_letters[1], 1):]) counties.append(converted_adress_name[ converted_adress_name. find(first_letters[0], 0):converted_adress_name .find(first_letters[1], 1)]) except: # index error # it will occur, if first_letters includes only one letter. # if district is not specified, put county name both district and county containers. counties.append( converted_adress_name[converted_adress_name. find(first_letters[0]):]) districts.append( converted_adress_name[converted_adress_name. find(first_letters[0]):]) # finding other data points like date of ad, number of rooms, total area, floor within building etc. # to do this, i will iterate through all of advertisement links for y in links_ad: start_time = time.time() ur = cm.request(y) tracker += 1 time.sleep(random.randint(2, 6)) requests += 1 elapsed_time = time.time() - start_time print( 'Request:{}; Frequency: {} requests/s; elapsed time:{}' .format(requests, requests / elapsed_time, elapsed_time)) soup = bs4.BeautifulSoup(ur.read(), 'lxml') info = soup.find('div', attrs={'class': 'classifiedInfo' }) # attributes of ads dct = { } # this will contain column names as keys, and data points as values. for v in info.findAll('ul'): head = v.find_all( 'strong' ) # name of columns. i.e. 'number_of_rooms' column. attribute = v.find_all( 'span' ) # value corresponds to above particular column. i.e. (3+1) for a, b in zip(head, attribute): columns = ' '.join(a.text.split()) data = ' '.join(b.text.split()) dct[columns] = data print(tracker) if tracker % 5 == 0: cm.new_identity( ) # After sending 5 requests, change identity in order to avoid being blocked. other_items.append(pd.DataFrame(data=dct, index=[0])) print('done') else: # If status code of request is not 200. break print("--- %s seconds ---" % (time.time() - strt)) return other_items, data_ids, ad_titles, ad_links, seller_names, seller_links, prices, counties, districts
class LoadIfileManager: WORK_MEM = 10240 def __init__(self, connectionStr): self.connMgr = ConnectionManager() self.conn = self.connMgr.connectToDatabase(connectionStr) self.cur = self.conn.cursor() self.fdm = ForeignDataManager() self.cur.execute("set work_mem to %s", (self.WORK_MEM, )) #print("Load IFile Manager Initialized.") def dropForeignTable(self, fdwTableName): self.cur.execute("drop foreign table if exists " + fdwTableName + ";") #print("drop foreign table if exists "+fdwTableName+";") def createForeignTable(self, iFile, fTableName): header, fdwScript = self.fdm.generateFDWScript(iFile, fTableName) self.cur.execute(fdwScript) return header def createFileWithoutDuplicatesV1(self, outputFilePath, noDupsSql): copyStmt = "copy (" + noDupsSql + ") to '" + outputFilePath + "' with delimiter E'\\t'" + " csv header;" #print("copyStmt = "+copyStmt) self.cur.execute(copyStmt) def createFileWithoutDuplicates(self, outputFilePath, noDupsSql): copyStmt = "copy (" + noDupsSql + ") to STDOUT with delimiter E'\\t'" + " csv header;" with open(outputFilePath, 'w') as outputFile: #let's try 20MB buffer size for a start, default was 8MB self.cur.copy_expert(copyStmt, outputFile, 20480) outputFile.close() def loadDataV1(self, tableName, header, fileToLoad, primaryKeyColumnName): loadSql = "copy " + tableName + " (" + ( ",".join(header) ) + ")" + " from '" + fileToLoad + "' with delimiter E'\\t' csv header;" #print("loadSql = "+loadSql) self.cur.execute(loadSql) self.updateSerialSequence(tableName, primaryKeyColumnName) def loadData(self, tableName, header, fileToLoad, primaryKeyColumnName): loadSql = "copy " + tableName + " (" + (",".join( header)) + ")" + " from STDIN with delimiter E'\\t' csv header;" rowsLoaded = 0 with open(fileToLoad, 'r') as f: self.cur.copy_expert(loadSql, f, 20480) rowsLoaded = self.cur.rowcount #print("Rows loaded = %s" % self.cur.rowcount) f.close() self.updateSerialSequence(tableName, primaryKeyColumnName) return rowsLoaded def upsertKVPFromForeignTable(self, fTableName, sourceKey, sourceValue, targetTable, targetId, targetJsonb): kvpSql = "select * from upsertKVPFromForeignTable('" + fTableName.lower( ) + "', '" + sourceKey + "', '" + sourceValue + "', '" + targetTable + "', '" + targetId + "', '" + targetJsonb + "');" #print ("kvpSQL: %s" % kvpSql) self.cur.execute(kvpSql) rowsLoaded = self.cur.fetchone() if rowsLoaded is not None: return rowsLoaded[0] else: return rowsLoaded def updateSerialSequence(self, tableName, primaryKeyColumnName): updateSeqSql = "SELECT pg_catalog.setval(pg_get_serial_sequence('" + tableName + "', '" + primaryKeyColumnName + "'), MAX(" + primaryKeyColumnName + ")) FROM " + tableName + ";" #print("updateSeqSql = "+updateSeqSql) self.cur.execute(updateSeqSql) def commitTransaction(self): self.conn.commit() def rollbackTransaction(self): self.conn.rollback() def closeConnection(self): self.connMgr.disconnectFromDatabase()
def __init__(self): try: # initiate logger self.logger = logging.getLogger() self.logger.setLevel(logging.DEBUG) self.log_send_store_handler = LogSendStoreHandler(LOG_LOCATION) formatter = logging.Formatter( '%(asctime)s - %(levelname)s - %(name)s - %(message)s') self.log_send_store_handler.setFormatter(formatter) self.logger.addHandler(self.log_send_store_handler) self.logger.info('Initialising system...') job_info_filter = JobInfoFilter() logging.getLogger('apscheduler.scheduler').addFilter( job_info_filter) logging.getLogger('apscheduler.threadpool').addFilter( job_info_filter) # load local configuration self.conf_man = ConfigurationManager(CONFIG_LOCATION) self.log_send_store_handler.update_configuration() self.scheduler = Scheduler() self.scheduler.start() self.packet_manager = PacketManager(self.scheduler) # initiate network connection self.connection = ConnectionManager() # add scheduler and connection to log handler self.log_send_store_handler.update_configuration( scheduler=self.scheduler, connection=self.connection) # try to connect connected_to_internet = self.connection.check_internet_connection() connected_to_server = self.connection.check_server_connection() if connected_to_internet and connected_to_server: self.load_online_configuration_and_initiate_sending_data() self.packet_manager.update_time() self.packet_manager.initiate_send_packets(self.connection) else: ''' if there is no connection: keep checking for a connection temporarily use offline timer and modbus slave configuration ''' if connected_to_internet: self.packet_manager.update_time() self.wait_for_connection_to_load_configuration() # initiate sensor timers self.read_sensor_scheduler = ReadSensorScheduler( self.scheduler, self.packet_manager) self.led_manager = LedManager(self.scheduler) self.led_manager.update_led(PinName.powered, LedState.on) self.set_up_led_manager_calls() # sleep 2 seconds to intialise led of log handler sleep(1) self.logger.info('Initialisation complete') while True: sleep(10) self.logger.debug('Alive and kicking') if self.logger.level is logging.DEBUG: scheduler_jobs = self.scheduler.get_jobs() if len(scheduler_jobs) > 1: self.logger.debug('Current scheduler jobs:') for index, job in enumerate(scheduler_jobs): self.logger.debug(' Job {0}: {1} {2}'.format( index, job.name, job.next_run_time)) else: self.logger.debug('No running scheduler jobs') except Exception as e: self.logger.error(e) raise self.log_send_store_handler.send_logs_job()
class Backdoor: def stop(self, signal=None, frame=None): if signal: self.logger.info('Caught SIGINT.') self.logger.info('Shutting down.') self.connection_manager.stop() self.running = False def __init__(self): self.logger = logging.getLogger('backdoor') self.logger.setLevel(logging.DEBUG) ch = logging.StreamHandler() ch.setLevel(logging.DEBUG) formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') ch.setFormatter(formatter) self.logger.addHandler(ch) self.logger.info('Starting backdoor.') signal.signal(signal.SIGINT, self.stop) self.running = True self.connection_manager = ConnectionManager(config.api_host, config.api_port) self.connection_manager.start() def run(self): while self.running: self.update() self.logger.info('Backdoor was shut down.') def update(self): try: query = self.connection_manager.queries.get(block=False) self.logger.debug('Got query:') self.logger.debug(query.query) if query.token in self.connection_manager.devices or query.token in self.connection_manager.webuis: self.handle_query(query) else: self.logger.info('A request from anon registered device was discarded:') self.logger.info(query.query) except queue.Empty: pass except Exception as e: self.logger.exception('Caught exception during the update process:') self.logger.exception(e) def issue_query(self, device, query): device = device if type(device) == str else device.pubkey if device in self.connection_manager.devices: self.connection_manager.devices[device].queries.put(query) elif device in self.connection_manager.webuis: self.connection_manager.webuis[device].queries.put(query) else: self.logger.info('Device or webui with token %s is not registered. Request was discarded:' % device) self.logger.info(query.query) def open(self, device): query = Query() query.create_open(config.server_token) self.issue_query(device, query) @helpers.handle_dbsession() def handle_query(session, self, query): response = Query() self.logger.debug('Handle query:') self.logger.debug(query.query) if query.method == 'ACCESS': token = session.query(Token).filter_by(value=query.params[0]).first() device = session.query(Device).filter_by(pubkey_device=query.token).first() if len(query.params) == 1: if token in device.tokens and token.expiry_date >= helpers.today(): response.create_grant(config.server_token, query.params[0]) self.logger.info('Granted access to token %s at device %s' % (query.params[0], query.token)) else: response.create_deny(config.server_token, query.params[0]) self.logger.info('Denied access to token %s at device %s' % (query.params[0], query.token)) self.issue_query(query.token, response) else: self.logger.debug('Broken query. Expected exactly 1 parameter.') elif query.method == 'FLASH': self.logger.info('Requested flash of token %s at device %s' % (query.params[0], query.params[1])) if len(query.params) == 2: if query.token in self.connection_manager.webuis: response.create_flash(config.server_token, query.params[0]) self.issue_query(query.params[1], response) else: self.logger.info('Requested flash came from a non webui or an unregistered one. It was discarded.') else: self.logger.debug('Broken query. Expected exactly 2 parameters.') elif query.method == 'FLASHED': if len(query.params) == 1: session.query(Token).filter_by(value=query.params[0]).first().flashed = True self.logger.debug('Token %s was flashed' % query.params[0]) self.logger.debug('Broken query. Expected exactly 1 parameter.') elif query.method == 'OPEN': if len(query.params) == 1: if query.token in self.connection_manager.webuis: response.create_open(config.server_token) self.issue_query(query.params[0], response) self.logger.debug('Sent OPEN to device with token %s.' % query.params[0]) else: self.logger.info('Requested flash came from a non webui or an unregistered one. It was discarded.') self.logger.debug('Broken query. Expected exactly 1 parameter.')
def __init__(self): self.connection_manager = ConnectionManager() env_location = pkg_resources.resource_filename('resources', '.env') if os.environ.get('DOTENV_LOADED', '0') != '1': load_dotenv(env_location) self.personal = os.environ.get("BLAKE2D_KEY", "topsecretkey").encode()
import json import os.path import time from sanic.log import logger from connection_manager import ConnectionManager log_params = sanic.log.LOGGING_CONFIG_DEFAULTS fname = time.asctime().replace(' ', '_').replace(':', '') + '.log' for h in log_params['handlers']: if 'stream' in log_params['handlers'][h]: del log_params['handlers'][h]['stream'] log_params['handlers'][h]['class'] = 'logging.FileHandler' log_params['handlers'][h]['filename'] = os.path.join( '..', 'server_logs', fname) CM = ConnectionManager() kv_server = sanic.Sanic(log_config=log_params) async def check_exist(kid): conn = await CM.get_connection() retval = (1, sanic.response.HTTPResponse(status=404)) try: logger.info(f'Searching for pair, key: "{kid}"') res = await conn.select('kv', key=[kid], index='primary') if len(res) != 0: logger.info( f'Found pair, key: "{kid}", value: "{res[0]["value"]}"') retval = (0, sanic.response.json(res[0]['value'])) else:
def hurriyetemlak_scraper(page_num): cm = ConnectionManager( ) # instance of connection manager object. This will be used for changing identity start_time = time.time() data = [ ] # list that contains all of the home attirbutes like numbe of rooms, building age, net area etc. last = page_num + 1 page_tracker = 0 for t in range(0, last): url = 'https://www.hurriyetemlak.com/antalya-kiralik/daire?page={}'.format( t) rr = cm.request(url) # status code of request. status_code = rr.code if rr != '' else -1 if status_code == 200: soup = bs4.BeautifulSoup(rr.read(), 'lxml') page_tracker += 1 # links = [] # for i in soup.findAll('div', attrs={'class':'list-item timeshare clearfix'}): # attrs = i.find('a') # links.append('https://www.hurriyetemlak.com'+attrs.get('href')) # above commented code is first verison of script for getting home advertisement links # this code is a map verison of that for speed issiue. links = list( map( lambda x: 'https://www.hurriyetemlak.com' + x.find('a'). get('href'), soup.findAll( 'div', attrs={'class': 'list-item timeshare clearfix'}))) else: print('Request failed.') break tracker = 0 requests = 0 # to track number of requesets that has been successfully sent to website. for _ in links: start_time = time.time() r = cm.request(_) tracker += 1 # in order to avoid overloading website, wait between 2 and 7 seconds for each iteration. time.sleep(random.randint(2, 7)) requests += 1 elapsed_time = time.time() - start_time print( 'Request:{}; Frequency: {} requests/s; elapsed time:{}'.format( requests, requests / elapsed_time, elapsed_time)) stat_code = r.code if r != '' else -1 # status code of request. if stat_code == 200: so = bs4.BeautifulSoup(r.read(), 'lxml') #lists of attributes of home ads that also includes html tags. raw_items = [ b.find_all('span') for f in so.findAll('li', attrs={'class': 'info-line'}) for b in f.findAll('li') ] else: print('Request failed') break print(tracker) if tracker % 5 == 0: cm.new_identity() revised_items = [] for r in raw_items: revised_items.append(r) # if this particular element is just empty list. if str(r) == '[]': break dct = { } # dictionary contains column names as keys and values of that columns as values. for i in revised_items[: -1]: # last element of revised_items list is just empty list. ignore it. try: # names of attributes corresponds to particular home ad. col = str(i[0]).split('>')[1].split('<')[0] # values of above attirbutes. row = str(i[1]).split('>')[1].split('<')[0] dct[col] = row # id_col and id_row are id of particular home ad. id_col = so.find('li', attrs={ 'class': 'realty-numb' }).text.replace('\n', '', 3).split(':')[0] id_row = so.find('li', attrs={ 'class': 'realty-numb' }).text.replace('\n', '', 3).split(':')[1] dct[id_col] = id_row # title of particular home ad. title = so.find('h1', attrs={ 'class': 'details-header' }).text dct['Title'] = title # price of particular home ad. price = so.find('li', attrs={ 'class': 'price-line clearfix' }).text.replace('\n', '', 3) dct['price'] = price except: continue # list of pandas dataframes. data.append(pd.DataFrame(dct, index=[0])) delays = [0.5, 1, 1.5, 2, 2.5, 3] time.sleep( np.random.choice(delays)) # to avoid overloading website. print("page {} is done".format(page_tracker)) print(time.time() - strt_time) return data
def __init__(self, connectionStr): self.connMgr = ConnectionManager() self.conn = self.connMgr.connectToDatabase(connectionStr) self.cur = self.conn.cursor() self.fdm = ForeignDataManager() self.cur.execute("set work_mem to %s", (self.WORK_MEM, ))
class Daemon(RegexMatchingEventHandler): # The path for configuration directory and daemon configuration file CONFIG_DIR = os.path.join(os.environ['HOME'], '.PyBox') CONFIG_FILEPATH = os.path.join(CONFIG_DIR, 'daemon_config') # Default configuration for Daemon, loaded if fail to load the config file from CONFIG_DIR DEF_CONF = OrderedDict() DEF_CONF['local_dir_state_path'] = os.path.join(CONFIG_DIR,'local_dir_state') DEF_CONF['sharing_path'] = os.path.join(os.environ['HOME'], 'sharing_folder') DEF_CONF['cmd_address'] = 'localhost' DEF_CONF['cmd_port'] = 50001 DEF_CONF['api_suffix'] = '/API/V1/' DEF_CONF['server_address'] = 'http://localhost:5000' DEF_CONF['user'] = '******' DEF_CONF['pass'] = '******' DEF_CONF['timeout_listener_sock'] = 0.5 DEF_CONF['backlog_listener_sock'] = 1 IGNORED_REGEX = ['.*\.[a-zA-z]+?#', # Libreoffice suite temporary file ignored '.*\.[a-zA-Z]+?~', # gedit issue solved ignoring this pattern: # gedit first delete file, create, and move to dest_path *.txt~ ] # Calculate int size in the machine architecture INT_SIZE = struct.calcsize('!i') def __init__(self): RegexMatchingEventHandler.__init__(self, ignore_regexes=Daemon.IGNORED_REGEX, ignore_directories=True) # Just Initialize variable the Daemon.start() do the other things self.daemon_state = 'down' # TODO implement the daemon state (disconnected, connected, syncronizing, ready...) self.running = 0 self.client_snapshot = {} # EXAMPLE {'<filepath1>: ['<timestamp>', '<md5>', '<filepath2>: ...} self.local_dir_state = {} # EXAMPLE {'last_timestamp': '<timestamp>', 'global_md5': '<md5>'} self.listener_socket = None self.observer = None self.cfg = self.load_cfg(Daemon.CONFIG_FILEPATH) self.init_sharing_path() self.conn_mng = ConnectionManager(self.cfg) def load_cfg(self, config_path): """ Load config, if impossible to find it or config file is corrupted restore it and load default configuration :param config_path: Path of config :return: dictionary containing configuration """ def build_default_cfg(): """ Restore default config file by writing on file :return: default configuration contained in the dictionary DEF_CONF """ with open(Daemon.CONFIG_FILEPATH, 'wb') as fo: json.dump(Daemon.DEF_CONF, fo, skipkeys=True, ensure_ascii=True, indent=4) return Daemon.DEF_CONF # Search if config directory exists otherwise create it if not os.path.isdir(Daemon.CONFIG_DIR): try: os.makedirs(Daemon.CONFIG_DIR) except (OSError, IOError): self.stop(1, '\nImpossible to create "{}" directory! Permission denied!\n'.format(Daemon.CONFIG_DIR)) if os.path.isfile(config_path): try: with open(config_path, 'r') as fo: loaded_config = json.load(fo) except ValueError: print '\nImpossible to read "{0}"! Config file overwrited and loaded default config!\n'.format(config_path) return build_default_cfg() corrupted_config = False for k in Daemon.DEF_CONF: if k not in loaded_config: corrupted_config = True # In the case is all gone right run config in loaded_config if not corrupted_config: return loaded_config else: print '\nWarning "{0}" corrupted! Config file overwrited and loaded default config!\n'.format(config_path) return build_default_cfg() else: print '\nWarning "{0}" doesn\'t exist, Config file overwrited and loaded default config!\n'.format(config_path) return build_default_cfg() def init_sharing_path(self): """ Check that the sharing folder exists otherwise create it. If is impossible to create exit with msg error. """ if not os.path.isdir(self.cfg['sharing_path']): try: os.makedirs(self.cfg['sharing_path']) except OSError: self.stop(1, '\nImpossible to create "{0}" directory! Check sharing_path value contained in the following file:\n"{1}"\n' .format(self.cfg['sharing_path'], Daemon.CONFIG_FILEPATH)) def build_client_snapshot(self): """ Build a snapshot of the sharing folder with the following structure self.client_snapshot { "<file_path>":('<timestamp>', '<md5>') } """ self.client_snapshot = {} for dirpath, dirs, files in os.walk(self.cfg['sharing_path']): for filename in files: filepath = os.path.join(dirpath, filename) unwanted_file = False for r in Daemon.IGNORED_REGEX: if re.match(r, filepath) is not None: unwanted_file = True print 'Ignored Path:', filepath break if not unwanted_file: relative_path = self.relativize_path(filepath) with open(filepath, 'rb') as f: self.client_snapshot[relative_path] = ['', hashlib.md5(f.read()).hexdigest()] def _is_directory_modified(self): if self.calculate_md5_of_dir() != self.local_dir_state['global_md5']: return True else: return False def search_md5(self, searched_md5): """ Recive as parameter the md5 of a file and return the first knowed path with the same md5 """ for path, tupla in self.client_snapshot.iteritems(): if searched_md5 in tupla[1]: return path else: return None def _sync_process(self, server_timestamp, server_dir_tree): # Makes the synchronization logic and return a list of commands to launch # for server synchronization def _filter_tree_difference(server_dir_tree): # process local dir_tree and server dir_tree # and makes a diffs classification # return a dict representing that classification # E.g. { 'new_on_server' : <[<filepath>, ...]>, # files in server, but not in client # 'modified' : <[<filepath>, ...]>, # files in server and client, but different # 'new_on_client' : <[<filepath>, ...]>, # files not in server, but in client # } client_files = set(self.client_snapshot.keys()) server_files = set(server_dir_tree.keys()) new_on_server = list(server_files.difference(client_files)) new_on_client = list(client_files.difference(server_files)) modified = [] for filepath in server_files.intersection(client_files): # check files md5 if server_dir_tree[filepath][1] != self.client_snapshot[filepath][1]: modified.append(filepath) return {'new_on_server': new_on_server, 'modified': modified, 'new_on_client': new_on_client} def _make_copy(src, dst): abs_src = self.absolutize_path(src) abs_dst = self.absolutize_path(dst) self.observer.skip(abs_dst) try: copy2(abs_src, abs_dst) except IOError: return False self.client_snapshot[dst] = self.client_snapshot[src] return True def _make_move(src, dst): abs_src = self.absolutize_path(src) abs_dst = self.absolutize_path(dst) self.observer.skip(abs_dst) try: move(abs_src, abs_dst) except IOError: return False self.client_snapshot[dst] = self.client_snapshot[src] self.client_snapshot.pop(src) return True def _check_md5(dir_tree, md5): for k, v in dir_tree.items(): if md5 == v[1]: return k return None local_timestamp = self.local_dir_state['last_timestamp'] tree_diff = _filter_tree_difference(server_dir_tree) sync_commands = [] if self._is_directory_modified(): if local_timestamp == server_timestamp: # simple case: the client has the command # it sends all folder modifications to server # files in server but not in client: remove them from server for filepath in tree_diff['new_on_server']: sync_commands.append(('delete', filepath)) #self.conn_mng.dispatch_request('delete', {'filepath': filepath}) # files modified in client: send modified files to server for filepath in tree_diff['modified']: sync_commands.append(('modified', filepath)) #self.conn_mng.dispatch_request('modified', {'filepath': filepath}) # files in client but not in server: upload them to server for filepath in tree_diff['new_on_client']: sync_commands.append(('upload', filepath)) #self.conn_mng.dispatch_request('upload', {'filepath': filepath}) else: # local_timestamp < server_timestamp # the server has the command for filepath in tree_diff['new_on_server']: timestamp, md5 = server_dir_tree[filepath] existed_filepath = _check_md5(self.client_snapshot, md5) if existed_filepath: # it's a copy or a move if _check_md5(server_dir_tree, md5): _make_copy(existed_filepath, filepath) else: _make_move(existed_filepath, filepath) tree_diff['new_on_client'].remove(filepath) else: if timestamp > local_timestamp: # the files in server is more updated sync_commands.append(('download', filepath)) #self.conn_mng.dispatch_request('download', {'filepath': filepath}) else: # the client has deleted the file, so delete it on server sync_commands.append(('delete', filepath)) #self.conn_mng.dispatch_request('delete', {'filepath': filepath}) for filepath in tree_diff['modified']: timestamp, md5 = server_dir_tree[filepath] if timestamp < local_timestamp: # the client has modified the file, so update it on server sync_commands.append(('modify', filepath)) #self.conn_mng.dispatch_request('modify', {'filepath': filepath}) else: # it's the worst case: # we have a conflict with server, # someone has modified files while daemon was down and someone else has modified # the same file on server conflicted_path = ''.join([filepath, '.conflicted']) _make_copy(filepath, conflicted_path) sync_commands.append(('upload', conflicted_path)) #self.conn_mng.dispatch_request('upload', {'filepath': conflicted_path}) for filepath in tree_diff['new_on_client']: sync_commands.append(('upload', filepath)) #self.conn_mng.dispatch_request('upload', {'filepath': filepath}) else: # directory not modified if local_timestamp == server_timestamp: # it's the best case. Client and server are already synchronized return [] else: # local_timestamp < server_timestamp # the server has the command for filepath in tree_diff['new_on_server']: timestamp, md5 = server_dir_tree[filepath] existed_filepath = _check_md5(self.client_snapshot, md5) if existed_filepath: # it's a copy or a move if _check_md5(server_dir_tree, md5): _make_copy(existed_filepath, filepath) else: _make_move(existed_filepath, filepath) tree_diff['new_on_client'].remove(filepath) else: # it's a new file sync_commands.append(('download', filepath)) #self.conn_mng.dispatch_request('download', {'filepath': filepath}) for filepath in tree_diff['modified']: sync_commands.append(('download', filepath)) #self.conn_mng.dispatch_request('download', {'filepath': filepath}) for filepath in tree_diff['new_on_client']: # files that have been deleted on server, so have to delete them abs_filepath = self.absolutize_path(filepath) self.observer.skip(abs_filepath) try: os.remove(abs_filepath) except OSError: # it should raise an exceptions pass self.client_snapshot.pop(filepath) return sync_commands def sync_with_server(self): """ Makes the synchronization with server """ response = self.conn_mng.dispatch_request('get_server_snapshot', '') if response is None: self.stop(1, '\nReceived bad snapshot. Server down?\n') server_timestamp = response['server_timestamp'] files = response['files'] sync_commands = self._sync_process(server_timestamp, files) self.update_local_dir_state(server_timestamp) # Initialize the variable where we put the timestamp of the last operation we did last_operation_timestamp = None # makes all synchronization commands for command, path in sync_commands: if command == 'delete': event_timestamp = self.conn_mng.dispatch_request(command, {'filepath': path}) if event_timestamp: print 'event_timestamp di "delete" INTO SYNC:', event_timestamp last_operation_timestamp = event_timestamp['server_timestamp'] # If i can't find path inside client_snapshot there is inconsistent problem in client_snapshot! if self.client_snapshot.pop(path, 'ERROR') == 'ERROR': print 'Error during delete event INTO SYNC! Impossible to find "{}" inside client_snapshot'.format(path) else: self.stop(1, 'Error during connection with the server. Server fail to "delete" this file: {}'.format(path)) elif command == 'modified' or command == 'upload': event_timestamp = self.conn_mng.dispatch_request(command, {'filepath': path}) if event_timestamp: print 'event_timestamp di "{}" INTO SYNC: {}'.format(command, event_timestamp) last_operation_timestamp = event_timestamp['server_timestamp'] else: self.stop(1, 'Error during connection with the server. Server fail to "{}" this file: {}'.format(command, path)) else: # command == 'download' print 'skip di download' self.observer.skip(self.absolutize_path(path)) connection_result = self.conn_mng.dispatch_request(command, {'filepath': path}) if connection_result: print 'Downloaded file with path "{}" INTO SYNC'.format(path) self.client_snapshot[path] = files[path] else: self.stop(1, 'Error during connection with the server. Client fail to "download" this file: {}'.format(path)) if last_operation_timestamp: self.update_local_dir_state(last_operation_timestamp) def relativize_path(self, abs_path): """ This function relativize the path watched by daemon: for example: /home/user/watched/subfolder/ will be subfolder/ """ if abs_path.startswith(self.cfg['sharing_path']): relative_path = abs_path[len(self.cfg['sharing_path']) + 1:] return relative_path else: raise Exception def absolutize_path(self, rel_path): """ This function absolutize a path that i have relativize before: for example: subfolder/ will be /home/user/watched/subfolder/ """ return os.path.join(self.cfg['sharing_path'], rel_path) def create_observer(self): """ Create an instance of the watchdog Observer thread class. """ self.observer = SkipObserver() self.observer.schedule(self, path=self.cfg['sharing_path'], recursive=True) # TODO handly erorrs in dictionary if the client_dispatcher miss required data!! # TODO update struct with new more performance data structure # TODO verify what happen if the server return a error message #################################### def on_created(self, e): def build_data(cmd, rel_new_path, new_md5, founded_path=None): """ Prepares the data from event handler to be delivered to connection_manager. """ data = {'cmd': cmd} if cmd == 'copy': data['file'] = {'src': founded_path, 'dst': rel_new_path, 'md5': new_md5, } else: data['file'] = {'filepath': rel_new_path, 'md5': new_md5, } return data new_md5 = self.hash_file(e.src_path) rel_new_path = self.relativize_path(e.src_path) founded_path = self.search_md5(new_md5) # with this check i found the copy events if founded_path: print 'start copy' data = build_data('copy', rel_new_path, new_md5, founded_path) # this elif check that this created aren't modified event elif rel_new_path in self.client_snapshot: print 'start modified FROM CREATE!!!!!' data = build_data('modify', rel_new_path, new_md5) else: # Finally we find a real create event! print 'start create' data = build_data('upload', rel_new_path, new_md5) # Send data to connection manager dispatcher and check return value. If all go right update client_snapshot and local_dir_state event_timestamp = self.conn_mng.dispatch_request(data['cmd'], data['file']) print 'event_timestamp di "{}" = {}'.format(data['cmd'], event_timestamp) if event_timestamp: self.client_snapshot[rel_new_path] = [event_timestamp, new_md5] self.update_local_dir_state(event_timestamp['server_timestamp']) else: self.stop(1, 'Impossible to connect with the server. Failed during "{0}" operation on "{1}" file' .format(data['cmd'], e.src_path )) def on_moved(self, e): print 'start move' rel_src_path = self.relativize_path(e.src_path) rel_dest_path = self.relativize_path(e.dest_path) # If i can't find rel_src_path inside client_snapshot there is inconsistent problem in client_snapshot! if self.client_snapshot.get(rel_src_path, 'ERROR') == 'ERROR': self.stop(1, 'Error during move event! Impossible to find "{}" inside client_snapshot'.format(rel_dest_path)) md5 = self.client_snapshot[rel_src_path][1] data = {'src': rel_src_path, 'dst': rel_dest_path, 'md5': md5, } # Send data to connection manager dispatcher and check return value. If all go right update client_snapshot and local_dir_state event_timestamp = self.conn_mng.dispatch_request('move', data) print 'event_timestamp di "move" =', event_timestamp if event_timestamp: self.client_snapshot[rel_dest_path] = [event_timestamp, md5] # I'm sure that rel_src_path exists inside client_snapshot because i check above so i don't check pop result self.client_snapshot.pop(rel_src_path) self.update_local_dir_state(event_timestamp['server_timestamp']) else: self.stop(1, 'Impossible to connect with the server. Failed during "move" operation on "{}" file'.format(e.src_path )) def on_modified(self, e): print 'start modified' new_md5 = self.hash_file(e.src_path) rel_path = self.relativize_path(e.src_path) data = {'filepath': rel_path, 'md5': new_md5 } # Send data to connection manager dispatcher and check return value. If all go right update client_snapshot and local_dir_state event_timestamp = self.conn_mng.dispatch_request('modify', data) if event_timestamp: print 'event_timestamp di "modified" =', event_timestamp self.client_snapshot[rel_path] = [event_timestamp, new_md5] self.update_local_dir_state(event_timestamp['server_timestamp']) else: self.stop(1, 'Impossible to connect with the server. Failed during "delete" operation on "{}" file'.format(e.src_path)) def on_deleted(self, e): print 'start delete' rel_deleted_path = self.relativize_path(e.src_path) # Send data to connection manager dispatcher and check return value. If all go right update client_snapshot and local_dir_state event_timestamp = self.conn_mng.dispatch_request('delete', {'filepath': rel_deleted_path}) if event_timestamp: print 'event_timestamp di "delete" =', event_timestamp # If i can't find rel_deleted_path inside client_snapshot there is inconsistent problem in client_snapshot! if self.client_snapshot.pop(rel_deleted_path, 'ERROR') == 'ERROR': print 'Error during delete event! Impossible to find "{}" inside client_snapshot'.format(rel_deleted_path) self.update_local_dir_state(event_timestamp['server_timestamp']) else: self.stop(1, 'Impossible to connect with the server. Failed during "delete" operation on "{}" file'.format(e.src_path)) def start(self): """ Starts the communication with the command_manager. """ self.build_client_snapshot() self.load_local_dir_state() # Operations necessary to start the daemon self.create_observer() self.observer.start() self.sync_with_server() self.listener_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) self.listener_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) self.listener_socket.bind((self.cfg['cmd_address'], self.cfg['cmd_port'])) self.listener_socket.listen(self.cfg['backlog_listener_sock']) r_list = [self.listener_socket] self.daemon_state = 'started' self.running = 1 polling_counter = 0 try: while self.running: r_ready, w_ready, e_ready = select.select(r_list, [], [], self.cfg['timeout_listener_sock']) for s in r_ready: if s == self.listener_socket: # handle the server socket client_socket, client_address = self.listener_socket.accept() r_list.append(client_socket) else: # handle all other sockets length = s.recv(Daemon.INT_SIZE) if length: # i need to do [0] and cast int because the struct.unpack return a tupla like (23234234,) # with the length as a string length = int(struct.unpack('!i', length)[0]) message = json.loads(s.recv(length)) for cmd, data in message.items(): if cmd == 'shutdown': raise KeyboardInterrupt self.conn_mng.dispatch_request(cmd, data) else: s.close() r_list.remove(s) # synchronization polling # makes the polling every 3 seconds, so it waits six cycle (0.5 * 6 = 3 seconds) # maybe optimizable but now functional polling_counter += 1 if polling_counter == 6: self.sync_with_server() polling_counter = 0 except KeyboardInterrupt: self.stop(0) self.observer.stop() self.observer.join() self.listener_socket.close() def stop(self, exit_status, exit_message=None): """ Stop the Daemon components (observer and communication with command_manager). """ if self.daemon_state == 'started': self.running = 0 self.daemon_state = 'down' self.save_local_dir_state() if exit_message: print exit_message exit(exit_status) def update_local_dir_state(self, last_timestamp): """ Update the local_dir_state with last_timestamp operation and save it on disk """ if isinstance(last_timestamp, int): self.local_dir_state['last_timestamp'] = last_timestamp self.local_dir_state['global_md5'] = self.calculate_md5_of_dir() self.save_local_dir_state() else: self.stop(1, 'Not int value assigned to local_dir_state[\'last_timestamp\']!\nIncorrect value: {}'.format(last_timestamp)) def save_local_dir_state(self): """ Save local_dir_state on disk """ json.dump(self.local_dir_state, open(self.cfg['local_dir_state_path'], "wb"), indent=4) print "local_dir_state saved" def load_local_dir_state(self): """ Load local dir state on self.local_dir_state variable if file doesn't exists it will be created without timestamp """ def _rebuild_local_dir_state(): self.local_dir_state = {'last_timestamp': 0.0, 'global_md5': self.calculate_md5_of_dir()} json.dump(self.local_dir_state, open(self.cfg['local_dir_state_path'], "wb"), indent=4) if os.path.isfile(self.cfg['local_dir_state_path']): self.local_dir_state = json.load(open(self.cfg['local_dir_state_path'], "rb")) if 'last_timestamp' in self.local_dir_state and 'global_md5' in self.local_dir_state \ and isinstance(self.local_dir_state['last_timestamp'], int): print "questo è last_timestamp:", self.local_dir_state['last_timestamp'] #self.local_dir_state['last_timestamp'] = int(self.local_dir_state['last_timestamp']) print "Loaded local_dir_state" else: print "local_dir_state corrupted. Reinitialized new local_dir_state" _rebuild_local_dir_state() else: print "local_dir_state not found. Initialize new local_dir_state" _rebuild_local_dir_state() def calculate_md5_of_dir(self, verbose=0): """ Calculate the md5 of the entire directory, with the md5 in client_snapshot and the md5 of full filepath string. When the filepath isn't in client_snapshot the md5 is calculated on fly :return is the md5 hash of the directory """ directory = self.cfg['sharing_path'] if verbose: start = time.time() md5Hash = hashlib.md5() if not os.path.exists(directory): self.stop(1, 'Error during calculate md5! Impossible to find "{}" in user folder'.format(directory)) for root, dirs, files in os.walk(directory, followlinks=False): for names in files: filepath = os.path.join(root, names) rel_path = self.relativize_path(filepath) if rel_path in self.client_snapshot: md5Hash.update(self.client_snapshot[rel_path][1]) md5Hash.update(hashlib.md5(filepath).hexdigest()) else: hashed_file = self.hash_file(filepath) if hashed_file: md5Hash.update(hashed_file) md5Hash.update(hashlib.md5(filepath).hexdigest()) else: print "can't hash file: ", filepath if verbose: stop = time.time() print stop - start return md5Hash.hexdigest() def hash_file(self, file_path, chunk_size=1024): """ :accept an absolute file path :return the md5 hash of received file """ md5Hash = hashlib.md5() try: f1 = open(file_path, 'rb') while 1: # Read file in as little chunks buf = f1.read(chunk_size) if not buf: break md5Hash.update(hashlib.md5(buf).hexdigest()) f1.close() return md5Hash.hexdigest() except (OSError, IOError) as e: print e return None
class AWSIOTDeviceAdapter(DeviceAdapter): """A device adapter allowing connections to devices over AWS IoT Args: port (string): A optional port string specifying a topic prefix to use if we are trying to connect to a gateway, otherwise, we assume that we're connecting directly to a device that is attached to AWS IoT. """ def __init__(self, port): super(AWSIOTDeviceAdapter, self).__init__() self.set_config('default_timeout', 5.0) reg = ComponentRegistry() endpoint = reg.get_config('awsiot-endpoint') rootcert = reg.get_config('awsiot-rootcert') iamuser = reg.get_config('awsiot-iamkey') iamsecret = reg.get_config('awsiot-iamtoken') iamsession = reg.get_config('awsiot-session', default=None) args = {} args['endpoint'] = endpoint args['root_certificate'] = rootcert args['use_websockets'] = True args['iam_key'] = iamuser args['iam_secret'] = iamsecret args['iam_session'] = iamsession self._logger = logging.getLogger(__name__) # Port should be a topic prefix that allows us to connect # only to subset of IOTile devices managed by a gateway # rather than to directly accessible iotile devices. if port is None: port = "" if len(port) > 0 and port[-1] != '/': port = port + '/' self.client = OrderedAWSIOTClient(args) self.name = str(uuid.uuid4()) self.client.connect(self.name) self.prefix = port self.conns = ConnectionManager(self.id) self.conns.start() self.client.subscribe(self.prefix + 'devices/+/data/advertisement', self._on_advertisement, ordered=False) self._deferred = queue.Queue() self.set_config('minimum_scan_time', 5.0) self.set_config('probe_supported', True) self.set_config('probe_required', True) self.mtu = self.get_config( 'mtu', 60 * 1024) # Split script payloads larger than this def connect_async(self, connection_id, connection_string, callback): """Connect to a device by its connection_string This function looks for the device on AWS IOT using the preconfigured topic prefix and looking for: <prefix>/devices/connection_string It then attempts to lock that device for exclusive access and returns a callback if successful. Args: connection_id (int): A unique integer set by the caller for referring to this connection once created connection_string (string): A device id of the form d--XXXX-YYYY-ZZZZ-WWWW callback (callable): A callback function called when the connection has succeeded or failed """ topics = MQTTTopicValidator(self.prefix + 'devices/{}'.format(connection_string)) key = self._generate_key() name = self.name conn_message = { 'type': 'command', 'operation': 'connect', 'key': key, 'client': name } context = {'key': key, 'slug': connection_string, 'topics': topics} self.conns.begin_connection(connection_id, connection_string, callback, context, self.get_config('default_timeout')) self._bind_topics(topics) try: self.client.publish(topics.connect, conn_message) except IOTileException: self._unbind_topics(topics) self.conns.finish_connection(connection_id, False, 'Failed to send connection message') def disconnect_async(self, conn_id, callback): """Asynchronously disconnect from a device that has previously been connected Args: conn_id (int): a unique identifier for this connection on the DeviceManager that owns this adapter. callback (callable): A function called as callback(conn_id, adapter_id, success, failure_reason) when the disconnection finishes. Disconnection can only either succeed or timeout. """ try: context = self.conns.get_context(conn_id) except ArgumentError: callback(conn_id, self.id, False, "Could not find connection information") return self.conns.begin_disconnection(conn_id, callback, self.get_config('default_timeout')) topics = context['topics'] disconn_message = { 'key': context['key'], 'client': self.name, 'type': 'command', 'operation': 'disconnect' } self.client.publish(topics.action, disconn_message) def send_script_async(self, conn_id, data, progress_callback, callback): """Asynchronously send a a script to this IOTile device Args: conn_id (int): A unique identifer that will refer to this connection data (string): the script to send to the device progress_callback (callable): A function to be called with status on our progress, called as: progress_callback(done_count, total_count) callback (callable): A callback for when we have finished sending the script. The callback will be called as callback(connection_id, adapter_id, success, failure_reason) 'connection_id': the connection id 'adapter_id': this adapter's id 'success': a bool indicating whether we received a response to our attempted RPC 'failure_reason': a string with the reason for the failure if success == False """ try: context = self.conns.get_context(conn_id) except ArgumentError: callback(conn_id, self.id, False, "Could not find connection information") return topics = context['topics'] context['progress_callback'] = progress_callback self.conns.begin_operation(conn_id, 'script', callback, 60.0) chunks = 1 if len(data) > self.mtu: chunks = len(data) // self.mtu if len(data) % self.mtu != 0: chunks += 1 # Send the script out possibly in multiple chunks if it's larger than our maximum transmit unit for i in xrange(0, chunks): start = i * self.mtu chunk = data[start:start + self.mtu] encoded = base64.standard_b64encode(chunk) script_message = { 'key': context['key'], 'client': self.name, 'type': 'command', 'operation': 'send_script', 'script': encoded, 'fragment_count': chunks, 'fragment_index': i } self.client.publish(topics.action, script_message) def send_rpc_async(self, conn_id, address, rpc_id, payload, timeout, callback): """Asynchronously send an RPC to this IOTile device Args: conn_id (int): A unique identifier that will refer to this connection address (int): the address of the tile that we wish to send the RPC to rpc_id (int): the 16-bit id of the RPC we want to call payload (bytearray): the payload of the command timeout (float): the number of seconds to wait for the RPC to execute callback (callable): A callback for when we have finished the RPC. The callback will be called as" callback(connection_id, adapter_id, success, failure_reason, status, payload) 'connection_id': the connection id 'adapter_id': this adapter's id 'success': a bool indicating whether we received a response to our attempted RPC 'failure_reason': a string with the reason for the failure if success == False 'status': the one byte status code returned for the RPC if success == True else None 'payload': a bytearray with the payload returned by RPC if success == True else None """ try: context = self.conns.get_context(conn_id) except ArgumentError: callback(conn_id, self.id, False, "Could not find connection information", 0xFF, bytearray()) return self.conns.begin_operation(conn_id, 'rpc', callback, timeout) topics = context['topics'] encoded_payload = binascii.hexlify(payload) rpc_message = { 'key': context['key'], 'client': self.name, 'type': 'command', 'operation': 'rpc', 'address': address, 'rpc_id': rpc_id, 'payload': encoded_payload, 'timeout': timeout } self.client.publish(topics.action, rpc_message) def _open_rpc_interface(self, conn_id, callback): """Enable RPC interface for this IOTile device Args: conn_id (int): the unique identifier for the connection callback (callback): Callback to be called when this command finishes callback(conn_id, adapter_id, success, failure_reason) """ self._open_interface(conn_id, 'rpc', callback) def _open_streaming_interface(self, conn_id, callback): """Enable streaming interface for this IOTile device Args: conn_id (int): the unique identifier for the connection callback (callback): Callback to be called when this command finishes callback(conn_id, adapter_id, success, failure_reason) """ self._open_interface(conn_id, 'streaming', callback) def _open_tracing_interface(self, conn_id, callback): """Enable tracing interface for this IOTile device Args: conn_id (int): the unique identifier for the connection callback (callback): Callback to be called when this command finishes callback(conn_id, adapter_id, success, failure_reason) """ self._open_interface(conn_id, 'tracing', callback) def _open_script_interface(self, conn_id, callback): """Enable script interface for this IOTile device Args: conn_id (int): the unique identifier for the connection callback (callback): Callback to be called when this command finishes callback(conn_id, adapter_id, success, failure_reason) """ self._open_interface(conn_id, 'script', callback) def _open_interface(self, conn_id, iface, callback): """Open an interface on this device Args: conn_id (int): the unique identifier for the connection iface (string): the interface name to open callback (callback): Callback to be called when this command finishes callback(conn_id, adapter_id, success, failure_reason) """ try: context = self.conns.get_context(conn_id) except ArgumentError: callback(conn_id, self.id, False, "Could not find connection information") return self.conns.begin_operation(conn_id, 'open_interface', callback, self.get_config('default_timeout')) topics = context['topics'] open_iface_message = { 'key': context['key'], 'type': 'command', 'operation': 'open_interface', 'client': self.name, 'interface': iface } self.client.publish(topics.action, open_iface_message) def stop_sync(self): """Synchronously stop this adapter """ conn_ids = self.conns.get_connections() # If we have any open connections, try to close them here before shutting down for conn in conn_ids: try: self.disconnect_sync(conn) except HardwareError: pass self.client.disconnect() self.conns.stop() def probe_async(self, callback): """Probe for visible devices connected to this DeviceAdapter. Args: callback (callable): A callback for when the probe operation has completed. callback should have signature callback(adapter_id, success, failure_reason) where: success: bool failure_reason: None if success is True, otherwise a reason for why we could not probe """ topics = MQTTTopicValidator(self.prefix) self.client.publish(topics.probe, { 'type': 'command', 'operation': 'probe', 'client': self.name }) callback(self.id, True, None) def periodic_callback(self): """Periodically help maintain adapter internal state """ while True: try: action = self._deferred.get(False) action() except queue.Empty: break except Exception: self._logger.exception('Exception in periodic callback') def _bind_topics(self, topics): """Subscribe to all the topics we need to communication with this device Args: topics (MQTTTopicValidator): The topic validator for this device that we are connecting to. """ # FIXME: Allow for these subscriptions to fail and clean up the previous ones # so that this function is atomic self.client.subscribe(topics.status, self._on_status_message) self.client.subscribe(topics.tracing, self._on_trace) self.client.subscribe(topics.streaming, self._on_report) self.client.subscribe(topics.response, self._on_response_message) def _unbind_topics(self, topics): """Unsubscribe to all of the topics we needed for communication with device Args: topics (MQTTTopicValidator): The topic validator for this device that we have connected to. """ self.client.unsubscribe(topics.status) self.client.unsubscribe(topics.tracing) self.client.unsubscribe(topics.streaming) self.client.unsubscribe(topics.response) def _generate_key(self): """Generate a random 32 byte key and encode it in hex Returns: string: Cryptographically random 64 character string """ key = os.urandom(32) return binascii.hexlify(key) def _find_connection(self, topic): """Attempt to find a connection id corresponding with a topic The device is found by assuming the topic ends in <slug>/[control|data]/channel Args: topic (string): The topic we received a message on Returns: int: The internal connect id (device slug) associated with this topic """ parts = topic.split('/') if len(parts) < 3: return None slug = parts[-3] return slug def _on_advertisement(self, sequence, topic, message): try: # FIXME: We need a global topic validator to validate these messages # message = self.topics.validate_message(['advertisement'], message_type, message) del message['operation'] del message['type'] self._trigger_callback( 'on_scan', self.id, message, 60.) # FIXME: Get the timeout from somewhere except IOTileException as exc: pass def _on_report(self, sequence, topic, message): """Process a report received from a device. Args: sequence (int): The sequence number of the packet received topic (string): The topic this message was received on message (dict): The message itself """ try: conn_key = self._find_connection(topic) conn_id = self.conns.get_connection_id(conn_key) except ArgumentError: self._logger.warn( "Dropping report message that does not correspond with a known connection, topic=%s", topic) return try: rep_msg = messages.ReportNotification.verify(message) serialized_report = {} serialized_report['report_format'] = rep_msg['report_format'] serialized_report['encoded_report'] = rep_msg['report'] serialized_report['received_time'] = datetime.datetime.strptime( rep_msg['received_time'].decode(), "%Y%m%dT%H:%M:%S.%fZ") report = IOTileReportParser.DeserializeReport(serialized_report) self._trigger_callback('on_report', conn_id, report) except Exception: self._logger.exception("Error processing report conn_id=%d", conn_id) def _on_trace(self, sequence, topic, message): """Process a trace received from a device. Args: sequence (int): The sequence number of the packet received topic (string): The topic this message was received on message (dict): The message itself """ try: conn_key = self._find_connection(topic) conn_id = self.conns.get_connection_id(conn_key) except ArgumentError: self._logger.warn( "Dropping trace message that does not correspond with a known connection, topic=%s", topic) return try: tracing = messages.TracingNotification.verify(message) self._trigger_callback('on_trace', conn_id, tracing['trace']) except Exception: self._logger.exception("Error processing trace conn_id=%d", conn_id) def _on_status_message(self, sequence, topic, message): """Process a status message received Args: sequence (int): The sequence number of the packet received topic (string): The topic this message was received on message (dict): The message itself """ self._logger.debug("Received message on (topic=%s): %s" % (topic, message)) try: conn_key = self._find_connection(topic) except ArgumentError: self._logger.warn( "Dropping message that does not correspond with a known connection, message=%s", message) return if messages.ConnectionResponse.matches(message): if self.name != message['client']: self._logger.debug( "Connection response received for a different client, client=%s, name=%s", message['client'], self.name) return self.conns.finish_connection(conn_key, message['success'], message.get('failure_reason', None)) else: self._logger.warn( "Dropping message that did not correspond with a known schema, message=%s", message) def _on_response_message(self, sequence, topic, message): """Process a response message received Args: sequence (int): The sequence number of the packet received topic (string): The topic this message was received on message (dict): The message itself """ try: conn_key = self._find_connection(topic) context = self.conns.get_context(conn_key) except ArgumentError: self._logger.warn( "Dropping message that does not correspond with a known connection, message=%s", message) return if 'client' in message and message['client'] != self.name: self._logger.debug( "Dropping message that is for another client %s, we are %s", message['client'], self.name) if messages.DisconnectionResponse.matches(message): self.conns.finish_disconnection( conn_key, message['success'], message.get('failure_reason', None)) elif messages.OpenInterfaceResponse.matches(message): self.conns.finish_operation(conn_key, message['success'], message.get('failure_reason', None)) elif messages.RPCResponse.matches(message): rpc_message = messages.RPCResponse.verify(message) self.conns.finish_operation( conn_key, rpc_message['success'], rpc_message.get('failure_reason', None), rpc_message.get('status', None), rpc_message.get('payload', None)) elif messages.ProgressNotification.matches(message): progress_callback = context.get('progress_callback', None) if progress_callback is not None: progress_callback(message['done_count'], message['total_count']) elif messages.ScriptResponse.matches(message): if 'progress_callback' in context: del context['progress_callback'] self.conns.finish_operation(conn_key, message['success'], message.get('failure_reason', None)) elif messages.DisconnectionNotification.matches(message): try: conn_key = self._find_connection(topic) conn_id = self.conns.get_connection_id(conn_key) except ArgumentError: self._logger.warn( "Dropping disconnect notification that does not correspond with a known connection, topic=%s", topic) return self.conns.unexpected_disconnect(conn_key) self._trigger_callback('on_disconnect', self.id, conn_id) else: self._logger.warn("Invalid response message received, message=%s", message)
def __init__(self, connection_manager=ConnectionManager()): self.connection_manager = connection_manager self.exchange_initialize() self.channel_initialize()
class Daemon(RegexMatchingEventHandler): # The path for configuration directory and daemon configuration file CONFIG_DIR = os.path.join(os.environ['HOME'], '.PyBox') CONFIG_FILEPATH = os.path.join(CONFIG_DIR, 'daemon_config') # Default configuration for Daemon, loaded if fail to load the config file from CONFIG_DIR DEF_CONF = OrderedDict() DEF_CONF['local_dir_state_path'] = os.path.join(CONFIG_DIR, 'local_dir_state') DEF_CONF['sharing_path'] = os.path.join(os.environ['HOME'], 'sharing_folder') DEF_CONF['cmd_address'] = 'localhost' DEF_CONF['cmd_port'] = 50001 DEF_CONF['api_suffix'] = '/API/V1/' DEF_CONF['server_address'] = 'http://localhost:5000' IGNORED_REGEX = ['.*\.[a-zA-z]+?#', # Libreoffice suite temporary file ignored '.*\.[a-zA-Z]+?~', # gedit issue solved ignoring this pattern: # gedit first delete file, create, and move to dest_path *.txt~ ] # Calculate int size in the machine architecture INT_SIZE = struct.calcsize('!i') # Allowed operation before user is activated ALLOWED_OPERATION = {'register', 'activate'} def __init__(self, cfg_path=None, sharing_path=None): RegexMatchingEventHandler.__init__(self, ignore_regexes=Daemon.IGNORED_REGEX, ignore_directories=True) # Just Initialize variable the Daemon.start() do the other things self.daemon_state = 'down' # TODO implement the daemon state (disconnected, connected, syncronizing, ready...) self.running = 0 self.client_snapshot = {} # EXAMPLE {'<filepath1>: ['<timestamp>', '<md5>', '<filepath2>: ...} self.local_dir_state = {} # EXAMPLE {'last_timestamp': '<timestamp>', 'global_md5': '<md5>'} self.listener_socket = None self.observer = None self.cfg = self._load_cfg(cfg_path, sharing_path) self._init_sharing_path(sharing_path) self.conn_mng = ConnectionManager(self.cfg) def _build_directory(self, path): """ Create a given directory if not existent :param path: the path of dir i want to create :return: boolean that indicate if the directory is now created or not. """ if not os.path.isdir(path): try: os.makedirs(path) except OSError: print '\nImpossible to create directory at the following path:\n{}\n'.format(path) return False else: print 'Created folder:\n', path return True def _create_cfg(self, cfg_path, sharing_path=None): """ Create the configuration file of client_daemon. If is given custom path for cfg (cfg_path) or observed directory (sharing_path) the config file will be updated with that configuration. If no cfg_path is given as default we save in default path stored in Daemon.CONFIG_FILEPATH. If no sharing_path is given as default we save in default path stored in Daemon.DEF_CONF['sharing_path']. :param cfg_path: Path of config :param sharing_path: Indicate the path of observed directory """ building_cfg = Daemon.DEF_CONF if cfg_path and cfg_path != Daemon.CONFIG_FILEPATH: Daemon.CONFIG_FILEPATH = cfg_path Daemon.CONFIG_DIR = os.path.dirname(cfg_path) building_cfg['local_dir_state_path'] = os.path.join(Daemon.CONFIG_DIR, 'local_dir_state') if sharing_path: building_cfg['sharing_path'] = sharing_path if self._build_directory(Daemon.CONFIG_DIR): with open(Daemon.CONFIG_FILEPATH, 'w') as daemon_config: json.dump(building_cfg, daemon_config, skipkeys=True, ensure_ascii=True, indent=4) return building_cfg else: self.stop(1, 'Impossible to create cfg file into {}'.format(Daemon.CONFIG_DIR)) def update_cfg(self): """ Update cfg with new state in self.cfg """ with open(Daemon.CONFIG_FILEPATH, 'w') as daemon_config: json.dump(self.cfg, daemon_config, skipkeys=True, ensure_ascii=True, indent=4) def _load_cfg(self, cfg_path, sharing_path): """ Load config, if impossible to find it or config file is corrupted restore it and load default configuration :param cfg_path: Path of config :param sharing_path: Indicate the path of observed directory :return: dictionary containing configuration """ if not cfg_path: cfg_path = Daemon.CONFIG_FILEPATH if os.path.isfile(cfg_path): try: with open(cfg_path, 'r') as fo: loaded_config = OrderedDict() for k, v in json.load(fo).iteritems(): loaded_config[k] = v except ValueError: print '\nImpossible to read "{0}"!' \ '\nConfig file overwrited and loaded with default configuration!\n'.format(cfg_path) else: # Check that all the key in DEF_CONF are in loaded_config if not [True for k in Daemon.DEF_CONF if k not in loaded_config]: # In the case is all gone right we can update the CONFIG costant and return loaded_config Daemon.CONFIG_FILEPATH = cfg_path Daemon.CONFIG_DIR = os.path.dirname(cfg_path) return loaded_config print '\nWarning "{0}" corrupted!\nConfig file overwrited and loaded with default configuration!\n'\ .format(cfg_path) else: print '\nWarning "{0}" doesn\'t exist!' \ '\nNew config file created and loaded with default configuration!\n'.format(cfg_path) return self._create_cfg(cfg_path, sharing_path) def _init_sharing_path(self, sharing_path): """ Check that the sharing folder exists otherwise create it. If is not given custom sharing_path we use default stored into self.cfg['sharing_path'] If is impossible to create the directory exit error message is given. """ if not sharing_path: sharing_path = Daemon.DEF_CONF['sharing_path'] if self._build_directory(sharing_path): self.cfg['sharing_path'] = sharing_path self.update_cfg() else: self.stop(1, '\nImpossible to create sharing folder in path:\n{}\n' 'Check sharing_path value contained in cfg file:\n{}\n' .format(self.cfg['sharing_path'], Daemon.CONFIG_FILEPATH)) def build_client_snapshot(self): """ Build a snapshot of the sharing folder with the following structure self.client_snapshot { "<file_path>":('<timestamp>', '<md5>') } """ self.client_snapshot = {} for dirpath, dirs, files in os.walk(self.cfg['sharing_path']): for filename in files: filepath = os.path.join(dirpath, filename) unwanted_file = False for r in Daemon.IGNORED_REGEX: if re.match(r, filepath) is not None: unwanted_file = True print 'Ignored Path:', filepath break if not unwanted_file: relative_path = self.relativize_path(filepath) self.client_snapshot[relative_path] = ['', self.hash_file(filepath)] def _is_directory_modified(self): """ The function check if the shared folder has been modified. It recalculate the md5 from client_snapshot and compares it with the global md5 stored in local_dir_state :return: True or False """ if self.md5_of_client_snapshot() != self.local_dir_state['global_md5']: return True else: return False def search_md5(self, searched_md5): """ Receive as parameter the md5 of a file and return the first knowed path with the same md5 """ for path, tupla in self.client_snapshot.iteritems(): if searched_md5 in tupla[1]: return path else: return None def _make_copy_on_client(self, src, dst, server_timestamp): """ Copy the file from src to dst if the dst already exists will be overwritten :param src: the relative path of source file to copy :param dst: the relative path of destination file to copy :return: True or False """ abs_src = self.absolutize_path(src) if not os.path.isfile(abs_src): return False abs_dst = self.absolutize_path(dst) dst_dir = os.path.dirname(abs_dst) if not os.path.isdir(dst_dir): os.makedirs(dst_dir) self.observer.skip(abs_dst) try: copy2(abs_src, abs_dst) except IOError as e: return False self.client_snapshot[dst] = self.client_snapshot[src] self.update_local_dir_state(server_timestamp) return True def _make_move_on_client(self, src, dst, server_timestamp): """ Move the file from src to dst. if the dst already exists will be overwritten :param src: the relative path of source file to move :param dst: the relative path of destination file to move :return: True or False """ abs_src = self.absolutize_path(src) if not os.path.isfile(abs_src): return False abs_dst = self.absolutize_path(dst) dst_dir = os.path.dirname(abs_dst) if not os.path.isdir(dst_dir): os.makedirs(dst_dir) self.observer.skip(abs_dst) try: move(abs_src, abs_dst) except IOError as e: return False self.client_snapshot[dst] = self.client_snapshot[src] self.client_snapshot.pop(src) self.update_local_dir_state(server_timestamp) return True def _sync_process(self, server_timestamp, server_dir_tree): # Makes the synchronization logic and return a list of commands to launch # for server synchronization def _filter_tree_difference(server_dir_tree): # process local dir_tree and server dir_tree # and makes a diffs classification # return a dict representing that classification # E.g. { 'new_on_server' : <[<filepath>, ...]>, # files in server, but not in client # 'modified' : <[<filepath>, ...]>, # files in server and client, but different # 'new_on_client' : <[<filepath>, ...]>, # files not in server, but in client # } client_files = set(self.client_snapshot.keys()) server_files = set(server_dir_tree.keys()) new_on_server = list(server_files.difference(client_files)) new_on_client = list(client_files.difference(server_files)) modified = [] for filepath in server_files.intersection(client_files): # check files md5 if server_dir_tree[filepath][1] != self.client_snapshot[filepath][1]: modified.append(filepath) return {'new_on_server': new_on_server, 'modified': modified, 'new_on_client': new_on_client} def _check_md5(dir_tree, md5): result = [] for k, v in dir_tree.iteritems(): if md5 == v[1]: result.append(k) return result local_timestamp = self.local_dir_state['last_timestamp'] tree_diff = _filter_tree_difference(server_dir_tree) sync_commands = [] if self._is_directory_modified(): if local_timestamp == server_timestamp: print "local_timestamp == server_timestamp and directory IS modified" # simple case: the client has the command # it sends all folder modifications to server # files in server but not in client: remove them from server for filepath in tree_diff['new_on_server']: sync_commands.append(('delete', filepath)) # self.conn_mng.dispatch_request('delete', {'filepath': filepath}) # files modified in client: send modified files to server for filepath in tree_diff['modified']: sync_commands.append(('modify', filepath)) # files in client but not in server: upload them to server for filepath in tree_diff['new_on_client']: sync_commands.append(('upload', filepath)) # self.conn_mng.dispatch_request('upload', {'filepath': filepath}) else: # local_timestamp < server_timestamp print "local_timestamp < server_timestamp and directory IS modified" assert local_timestamp <= server_timestamp, 'e\' successo qualcosa di brutto nella sync, ' \ 'local_timestamp > di server_timestamp ' # the server has the command for filepath in tree_diff['new_on_server']: file_timestamp, md5 = server_dir_tree[filepath] existed_filepaths_on_client = _check_md5(self.client_snapshot, md5) # If i found at least one path in client_snapshot with the same md5 of filepath this mean that in the past # client_snapshot have stored one or more files with the same md5 but different paths. if existed_filepaths_on_client: # it's a copy or a move for path in existed_filepaths_on_client: if path in tree_diff['new_on_client']: if self._make_move_on_client(path, filepath, server_timestamp): tree_diff['new_on_client'].remove(path) break else: self.stop(0, "move failed on in SYNC: src_path: {}, dest_path: {}".format(path, filepath)) # we haven't found files deleted on server so it's a copy else: if not self._make_copy_on_client(path, filepath, server_timestamp): self.stop(0, "copy failed on in SYNC: src_path: {}, dest_path: {}".format(path, filepath)) # the daemon don't know filepath, i will search if the file_timestamp is more recent then local_timestamp else: if file_timestamp > local_timestamp: # the files in server is more updated sync_commands.append(('download', filepath)) # self.conn_mng.dispatch_request('download', {'filepath': filepath}) else: # the client has deleted the file, so delete it on server sync_commands.append(('delete', filepath)) # self.conn_mng.dispatch_request('delete', {'filepath': filepath}) for filepath in tree_diff['modified']: file_timestamp, md5 = server_dir_tree[filepath] if file_timestamp < local_timestamp: # the client has modified the file, so update it on server sync_commands.append(('modify', filepath)) # self.conn_mng.dispatch_request('modify', {'filepath': filepath}) else: # it's the worst case: # we have a conflict with server, # someone has modified files while daemon was down and someone else has modified # the same file on server conflicted_path = ''.join([filepath, '.conflicted']) self._make_copy_on_client(filepath, conflicted_path, server_timestamp) sync_commands.append(('upload', conflicted_path)) # self.conn_mng.dispatch_request('upload', {'filepath': conflicted_path}) for filepath in tree_diff['new_on_client']: sync_commands.append(('upload', filepath)) # self.conn_mng.dispatch_request('upload', {'filepath': filepath}) else: # directory not modified if local_timestamp == server_timestamp: print "local_timestamp == server_timestamp and directory IS NOT modified" # it's the best case. Client and server are already synchronized return [] else: # local_timestamp < server_timestamp print "local_timestamp < server_timestamp and directory IS NOT modified" assert local_timestamp <= server_timestamp, 'e\' successo qualcosa di brutto nella sync, ' \ 'local_timestamp > di server_timestamp ' # the server has the command for filepath in tree_diff['new_on_server']: timestamp, md5 = server_dir_tree[filepath] existed_filepaths_on_client = _check_md5(self.client_snapshot, md5) # If i found at least one path in client_snapshot with the same md5 of filepath this mean that # in the past client_snapshot have stored one or more files with the same md5 but different paths. if existed_filepaths_on_client: # it's a copy or a move for path in existed_filepaths_on_client: if path in tree_diff['new_on_client']: if self._make_move_on_client(path, filepath, server_timestamp): tree_diff['new_on_client'].remove(path) break else: self.stop(0, "move failed on in SYNC: src_path: {}, dest_path: {}".format(path, filepath)) # we haven't found files deleted on server so it's a copy else: if not self._make_copy_on_client(path, filepath, server_timestamp): self.stop(0, "copy failed on in SYNC: src_path: {}, dest_path: {}".format(path, filepath)) else: # it's a new file sync_commands.append(('download', filepath)) # self.conn_mng.dispatch_request('download', {'filepath': filepath}) for filepath in tree_diff['modified']: sync_commands.append(('download', filepath)) # self.conn_mng.dispatch_request('download', {'filepath': filepath}) for filepath in tree_diff['new_on_client']: # files that have been deleted on server, so have to delete them abs_filepath = self.absolutize_path(filepath) self.observer.skip(abs_filepath) try: os.remove(abs_filepath) except OSError as e: print "Delete EXEPTION INTO SYNC : {}".format(e) self.client_snapshot.pop(filepath) self.update_local_dir_state(server_timestamp) return sync_commands def sync_with_server(self): """ Makes the synchronization with server """ response = self.conn_mng.dispatch_request('get_server_snapshot', '') if response is None: self.stop(1, '\nReceived None snapshot. Server down?\n') server_timestamp = response['server_timestamp'] files = response['files'] sync_commands = self._sync_process(server_timestamp, files) # Initialize the variable where we put the timestamp of the last operation we did last_operation_timestamp = server_timestamp # makes all synchronization commands for command, path in sync_commands: if command == 'delete': event_timestamp = self.conn_mng.dispatch_request(command, {'filepath': path}) if event_timestamp: last_operation_timestamp = event_timestamp['server_timestamp'] # If i can't find path inside client_snapshot there is inconsistent problem in client_snapshot! if self.client_snapshot.pop(path, 'ERROR') == 'ERROR': print 'Error during delete event INTO SYNC! Impossible to find "{}" inside client_snapshot'\ .format(path) else: self.stop(1, 'Error during connection with the server. Server fail to "delete" this file: {}'.format( path)) elif command == 'modify' or command == 'upload': new_md5 = self.hash_file(self.absolutize_path(path)) event_timestamp = self.conn_mng.dispatch_request(command, {'filepath': path, 'md5': new_md5}) if event_timestamp: last_operation_timestamp = event_timestamp['server_timestamp'] else: self.stop(1, 'Error during connection with the server. Server fail to "{}" this file: {}'.format( command, path)) else: # command == 'download' print 'skip di download' self.observer.skip(self.absolutize_path(path)) connection_result = self.conn_mng.dispatch_request(command, {'filepath': path}) if connection_result: print 'Downloaded file with path "{}" INTO SYNC'.format(path) self.client_snapshot[path] = files[path] else: self.stop(1, 'Error during connection with the server. Client fail to "download" this file: {}'.format( path)) self.update_local_dir_state(last_operation_timestamp) def relativize_path(self, abs_path): """ This function relativize the path watched by daemon: for example: /home/user/watched/subfolder/ will be subfolder/ """ if abs_path.startswith(self.cfg['sharing_path']): relative_path = abs_path[len(self.cfg['sharing_path']) + 1:] return relative_path else: raise Exception def absolutize_path(self, rel_path): """ This function absolutize a path that i have relativize before: for example: subfolder/ will be /home/user/watched/subfolder/ """ return os.path.join(self.cfg['sharing_path'], rel_path) def create_observer(self): """ Create an instance of the watchdog Observer thread class. """ self.observer = SkipObserver() self.observer.schedule(self, path=self.cfg['sharing_path'], recursive=True) # TODO handly erorrs in dictionary if the client_dispatcher miss required data!! # TODO update struct with new more performance data structure # TODO verify what happen if the server return a error message # ################################### def on_created(self, e): def build_data(cmd, rel_new_path, new_md5, founded_path=None): """ Prepares the data from event handler to be delivered to connection_manager. """ data = {'cmd': cmd} if cmd == 'copy': data['file'] = {'src': founded_path, 'dst': rel_new_path, 'md5': new_md5, } else: data['file'] = {'filepath': rel_new_path, 'md5': new_md5, } return data new_md5 = self.hash_file(e.src_path) rel_new_path = self.relativize_path(e.src_path) founded_path = self.search_md5(new_md5) # with this check i found the copy events if founded_path: print 'start copy' data = build_data('copy', rel_new_path, new_md5, founded_path) # this elif check that this created aren't modified event elif rel_new_path in self.client_snapshot: print 'start modified FROM CREATE!!!!!' data = build_data('modify', rel_new_path, new_md5) else: # Finally we find a real create event! print 'start create' data = build_data('upload', rel_new_path, new_md5) # Send data to connection manager dispatcher and check return value. # If all go right update client_snapshot and local_dir_state event_timestamp = self.conn_mng.dispatch_request(data['cmd'], data['file']) print 'event_timestamp di "{}" = {}'.format(data['cmd'], event_timestamp) if event_timestamp: self.client_snapshot[rel_new_path] = [event_timestamp, new_md5] self.update_local_dir_state(event_timestamp['server_timestamp']) else: self.stop(1, 'Impossible to connect with the server. Failed during "{0}" operation on "{1}" file' .format(data['cmd'], e.src_path)) def on_moved(self, e): print 'start move' rel_src_path = self.relativize_path(e.src_path) rel_dest_path = self.relativize_path(e.dest_path) # If i can't find rel_src_path inside client_snapshot there is inconsistent problem in client_snapshot! if self.client_snapshot.get(rel_src_path, 'ERROR') == 'ERROR': self.stop(1, 'Error during move event! Impossible to find "{}" inside client_snapshot'.format(rel_dest_path)) md5 = self.client_snapshot[rel_src_path][1] data = {'src': rel_src_path, 'dst': rel_dest_path, 'md5': md5, } # Send data to connection manager dispatcher and check return value. # If all go right update client_snapshot and local_dir_state event_timestamp = self.conn_mng.dispatch_request('move', data) print 'event_timestamp di "move" =', event_timestamp if event_timestamp: self.client_snapshot[rel_dest_path] = [event_timestamp, md5] # I'm sure that rel_src_path exists inside client_snapshot because i check above so i don't check pop result self.client_snapshot.pop(rel_src_path) self.update_local_dir_state(event_timestamp['server_timestamp']) else: self.stop(1, 'Impossible to connect with the server. Failed during "move" operation on "{}" file'.format( e.src_path)) def on_modified(self, e): print 'start modified' new_md5 = self.hash_file(e.src_path) rel_path = self.relativize_path(e.src_path) data = {'filepath': rel_path, 'md5': new_md5 } # Send data to connection manager dispatcher and check return value. # If all go right update client_snapshot and local_dir_state event_timestamp = self.conn_mng.dispatch_request('modify', data) if event_timestamp: print 'event_timestamp di "modified" =', event_timestamp self.client_snapshot[rel_path] = [event_timestamp, new_md5] self.update_local_dir_state(event_timestamp['server_timestamp']) else: self.stop(1, 'Impossible to connect with the server. Failed during "delete" operation on "{}" file'.format( e.src_path)) def on_deleted(self, e): print 'start delete' rel_deleted_path = self.relativize_path(e.src_path) # Send data to connection manager dispatcher and check return value. # If all go right update client_snapshot and local_dir_state event_timestamp = self.conn_mng.dispatch_request('delete', {'filepath': rel_deleted_path}) if event_timestamp: print 'event_timestamp di "delete" =', event_timestamp # If i can't find rel_deleted_path inside client_snapshot there is inconsistent problem in client_snapshot! if self.client_snapshot.pop(rel_deleted_path, 'ERROR') == 'ERROR': print 'Error during delete event! Impossible to find "{}" inside client_snapshot'.format( rel_deleted_path) self.update_local_dir_state(event_timestamp['server_timestamp']) else: self.stop(1, 'Impossible to connect with the server. Failed during "delete" operation on "{}" file'.format( e.src_path)) def _get_cmdmanager_request(self, socket): """ Communicate with cmd_manager and get the request Returns the request decoded by json format or None if cmd_manager send connection closure """ packet_size = socket.recv(Daemon.INT_SIZE) if len(packet_size) == Daemon.INT_SIZE: packet_size = int(struct.unpack('!i', packet_size)[0]) packet = '' remaining_size = packet_size while len(packet) < packet_size: packet_buffer = socket.recv(remaining_size) remaining_size -= len(packet_buffer) packet = ''.join([packet, packet_buffer]) req = json.loads(packet) return req else: return None def _set_cmdmanager_response(self, socket, message): """ Makes cmd_manager response encoding it in json format and send it to cmd_manager """ response = {'message': message} response_packet = json.dumps(response) socket.sendall(struct.pack('!i', len(response_packet))) socket.sendall(response_packet) return response_packet def _initialize_observing(self): """ Intial operation for observing. We create the client_snapshot, load the information stored inside local_dir_state and create observer. """ self.build_client_snapshot() self.load_local_dir_state() self.create_observer() self.observer.start() self.sync_with_server() def _activation_check(self, s, cmd, data): """ This method allow only registration and activation of user until this will be accomplished. In case of bad cmd this will be refused otherwise if the server response are successful we update the daemon_config and after activation of user start the observing. :param s: connection socket with client_cmdmanager :param cmd: received cmd from client_cmdmanager :param data: received data from client_cmdmanager """ if cmd not in Daemon.ALLOWED_OPERATION: self._set_cmdmanager_response(s, 'Operation not allowed! Authorization required.') else: response = self.conn_mng.dispatch_request(cmd, data) if response['successful']: if cmd == 'register': self.cfg['user'] = data[0] self.cfg['pass'] = data[1] self.update_cfg() elif cmd == 'activate': self.cfg['activate'] = True # Update the information about cfg into connection manager self.conn_mng.load_cfg(self.cfg) self.update_cfg() # Now the client_daemon is ready to operate, we do the start activity self._initialize_observing() self._set_cmdmanager_response(s, response) def start(self): """ Starts the communication with the command_manager. """ # If user is activated we can start observing. if self.cfg.get('activate', False): self._initialize_observing() TIMEOUT_LISTENER_SOCK = 0.5 BACKLOG_LISTENER_SOCK = 1 self.listener_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) self.listener_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) self.listener_socket.bind((self.cfg['cmd_address'], self.cfg['cmd_port'])) self.listener_socket.listen(BACKLOG_LISTENER_SOCK) r_list = [self.listener_socket] self.daemon_state = 'started' self.running = 1 polling_counter = 0 try: while self.running: r_ready, w_ready, e_ready = select.select(r_list, [], [], TIMEOUT_LISTENER_SOCK) for s in r_ready: if s == self.listener_socket: # handle the server socket client_socket, client_address = self.listener_socket.accept() r_list.append(client_socket) else: # handle all other sockets req = self._get_cmdmanager_request(s) if req: for cmd, data in req.iteritems(): if cmd == 'shutdown': self._set_cmdmanager_response(s, 'Deamon is shuting down') raise KeyboardInterrupt else: if not self.cfg.get('activate', False): self._activation_check(s, cmd, data) else: # client is already activated response = self.conn_mng.dispatch_request(cmd, data) # for now the protocol is that for request sent by # command manager, the server reply with a string # so, to maintain the same data structure during # daemon and cmdmanager comunications, it rebuild a json # to send like response # TODO it's advisable to make attention to this assertion or refact the architecture self._set_cmdmanager_response(s, response) else: # it receives the FIN packet that close the connection s.close() r_list.remove(s) if self.cfg.get('activate', False): # synchronization polling # makes the polling every 3 seconds, so it waits six cycle (0.5 * 6 = 3 seconds) # maybe optimizable but now functional polling_counter += 1 if polling_counter == 6: polling_counter = 0 self.sync_with_server() except KeyboardInterrupt: self.stop(0) if self.cfg.get('activate', False): self.observer.stop() self.observer.join() self.listener_socket.close() def stop(self, exit_status, exit_message=None): """ Stop the Daemon components (observer and communication with command_manager). """ if self.daemon_state == 'started': self.running = 0 self.daemon_state = 'down' if self.local_dir_state: self.save_local_dir_state() if exit_message: print exit_message exit(exit_status) def update_local_dir_state(self, last_timestamp): """ Update the local_dir_state with last_timestamp operation and save it on disk """ self.local_dir_state['last_timestamp'] = last_timestamp self.local_dir_state['global_md5'] = self.md5_of_client_snapshot() self.save_local_dir_state() def save_local_dir_state(self): """ Save local_dir_state on disk """ json.dump(self.local_dir_state, open(self.cfg['local_dir_state_path'], "w"), indent=4) print "local_dir_state saved" def load_local_dir_state(self): """ Load local dir state on self.local_dir_state variable if file doesn't exists it will be created without timestamp """ def _rebuild_local_dir_state(): self.local_dir_state = {'last_timestamp': 0, 'global_md5': self.md5_of_client_snapshot()} json.dump(self.local_dir_state, open(self.cfg['local_dir_state_path'], "w"), indent=4) if os.path.isfile(self.cfg['local_dir_state_path']): self.local_dir_state = json.load(open(self.cfg['local_dir_state_path'], "r")) print "Loaded local_dir_state" else: print "local_dir_state not found. Initialize new local_dir_state" _rebuild_local_dir_state() def md5_of_client_snapshot(self, verbose=0): """ Calculate the md5 of the entire directory snapshot, with the md5 in client_snapshot and the md5 of full filepath string. :return is the md5 hash of the directory """ if verbose: start = time.time() md5Hash = hashlib.md5() for path, time_md5 in sorted(self.client_snapshot.iteritems()): # extract md5 from tuple. we don't need hexdigest it's already md5 if verbose: print path md5Hash.update(time_md5[1]) md5Hash.update(path) if verbose: stop = time.time() print stop - start return md5Hash.hexdigest() def hash_file(self, file_path, chunk_size=1024): """ :accept an absolute file path :return the md5 hash of received file """ md5Hash = hashlib.md5() try: f1 = open(file_path, 'rb') while 1: # Read file in as little chunks buf = f1.read(chunk_size) if not buf: break md5Hash.update(buf) f1.close() return md5Hash.hexdigest() except (OSError, IOError) as e: print e return None
class PreprocessIfileManager: WORK_MEM = 10240 def __init__(self, connectionStr): self.connMgr = ConnectionManager() self.conn = self.connMgr.connectToDatabase(connectionStr) self.cur = self.conn.cursor() self.fdm = ForeignDataManager() self.cur.execute("set work_mem to %s", (self.WORK_MEM, )) #print("Preprocess IFile Manager Initialized.") def getCvIdOfTerm(self, term): self.cur.execute("select cv_id from cv where lower(term)=%s", (term.lower(), )) cv_id = self.cur.fetchone() if cv_id is not None: return cv_id[0] else: return cv_id def getCvIdOfGroupAndTerm(self, group, term): self.cur.execute( "select cv_id from cv where lower(\"group\")=%s and lower(term)=%s", (group.lower(), term.lower())) cv_id = self.cur.fetchone() if cv_id is not None: return cv_id[0] else: return cv_id #select data_type from information_schema.columns where table_name='marker' and column_name='platform_id'; def getTypeOfColumn(self, table, column): self.cur.execute( "select data_type from information_schema.columns where table_name=%s and column_name=%s", (table, column)) res = self.cur.fetchone() if res is not None: return res[0] else: return res def getColumnListOfTable(self, table): self.cur.execute( "select column_name from information_schema.columns where table_name = %s", (table, )) res = self.cur.fetchall() return res def dropForeignTable(self, fdwTableName): self.cur.execute("drop foreign table if exists " + fdwTableName) def createForeignTable(self, iFile, fTableName): header, fdwScript = self.fdm.generateFDWScript(iFile, fTableName) #print("fdwScript: %s" % fdwScript) self.cur.execute(fdwScript) return header def createFileWithDerivedIdsV1(self, outputFilePath, derivedIdSql): copyStmt = "copy (" + derivedIdSql + ") to '" + outputFilePath + "' with delimiter E'\\t'" + " csv header;" #print("copyStmt = "+copyStmt) self.cur.execute(copyStmt) def createFileWithDerivedIds(self, outputFilePath, derivedIdSql): copyStmt = "copy (" + derivedIdSql + ") to STDOUT with delimiter E'\\t'" + " csv header;" with open(outputFilePath, 'w') as outputFile: #let's try 20MB buffer size for a start, default was 8MB self.cur.copy_expert(copyStmt, outputFile, 20480) outputFile.close() def commitTransaction(self): self.conn.commit() def rollbackTransaction(self): self.conn.rollback() def closeConnection(self): self.connMgr.disconnectFromDatabase()
from background_monitor import BackgroundMonitor from database import Database from connection_manager import ConnectionManager from config import SERVER_URL if __name__ == '__main__': connection_manager = ConnectionManager(SERVER_URL) connection_manager.authorize() database = Database(connection_manager) background_monitor = BackgroundMonitor(database) background_monitor.run()
class TestConnectionManager(unittest.TestCase): def setUp(self): httpretty.enable() create_environment() make_fake_dir() with open(CONFIG_FILEPATH, 'r') as fo: self.cfg = json.load(fo) self.auth = (self.cfg['user'], self.cfg['pass']) self.base_url = ''.join([self.cfg['server_address'], self.cfg['api_suffix']]) self.files_url = ''.join([self.base_url, 'files/']) self.actions_url = ''.join([self.base_url, 'actions/']) self.shares_url = ''.join([self.base_url, 'shares/']) self.user_url = ''.join([self.base_url, 'users/']) self.cm = ConnectionManager(self.cfg) def tearDown(self): httpretty.disable() httpretty.reset() remove_fake_dir() @httpretty.activate def test_register_user(self): """ Test register user api: method = POST resource = <user> data = password=<password> """ data = (USR, PW) url = ''.join((self.user_url, USR)) content = 'user created' content_jsoned = json.dumps(content) httpretty.register_uri(httpretty.POST, url, status=200, body=content_jsoned) response = self.cm.do_register(data) self.assertIn('content', response) self.assertEqual(response['content'], content) self.assertTrue(response['successful']) @httpretty.activate def test_register_user_with_weak_password(self): """ Test register user api with weak password: method = POST resource = <user> data = password=<password> """ weak_password = '******' data = (USR, weak_password) url = ''.join((self.user_url, USR)) content = {'type_of_improvement': 'improvement suggested'} content_jsoned = json.dumps(content) httpretty.register_uri(httpretty.POST, url, status=403, body=content_jsoned) response = self.cm.do_register(data) self.assertIn('improvements', response) self.assertEqual(response['improvements'], content) self.assertFalse(response['successful']) @httpretty.activate def test_register_user_with_already_existent_user(self): """ Test register user api with already existent user: method = POST resource = <user> data = password=<password> """ data = (USR, PW) url = ''.join((self.user_url, USR)) # This is the only case where server doesn't send data with the message error httpretty.register_uri(httpretty.POST, url, status=409) response = self.cm.do_register(data) self.assertIn('content', response) self.assertIsInstance(response['content'], str) self.assertFalse(response['successful']) @httpretty.activate def test_fail_to_register_user(self): """ Test failed register request Test activate user api: method = POST resource = <user> data = password=<password> """ data = (USR, PW) url = ''.join((self.user_url, USR)) httpretty.register_uri(httpretty.POST, url, status=500) response = self.cm.do_register(data) self.assertIsInstance(response['content'], str) self.assertFalse(response['successful']) @httpretty.activate def test_activate_user(self): """ Test successful activation Test activate user api: method = PUT resource = <user> data = activation_code=<token> """ user = '******' token = '6c9fb345c317ad1d31ab9d6445d1a820' data = (user, token) url = ''.join((self.user_url, user)) answer = 'user activated' answer_jsoned = json.dumps(answer) httpretty.register_uri(httpretty.PUT, url, status=201, body=answer_jsoned) response = self.cm.do_activate(data) self.assertIsInstance(response['content'], unicode) self.assertTrue(response['successful']) @httpretty.activate def test_activate_user_already_existent(self): """ Test activate user already existent Test activate user api: method = PUT resource = <user> data = activation_code=<token> """ user = '******' token = 'bad_token' data = (user, token) url = ''.join((self.user_url, user)) httpretty.register_uri(httpretty.PUT, url, status=409) response = self.cm.do_activate(data) self.assertIsInstance(response['content'], str) self.assertFalse(response['successful']) @httpretty.activate def test_activate_user_not_existent(self): """ Test activate user not existent Test activate user api: method = PUT resource = <user> data = activation_code=<token> """ user = '******' token = 'bad_token' data = (user, token) url = ''.join((self.user_url, user)) httpretty.register_uri(httpretty.PUT, url, status=404) response = self.cm.do_activate(data) self.assertIsInstance(response['content'], str) self.assertFalse(response['successful']) @httpretty.activate def test_fail_to_activate_user(self): """ Test failed activation request Test activate user api: method = PUT resource = <user> data = activation_code=<token> """ user = '******' token = 'bad_token' data = (user, token) url = ''.join((self.user_url, user)) httpretty.register_uri(httpretty.PUT, url, status=500) response = self.cm.do_activate(data) self.assertIsInstance(response['content'], str) self.assertFalse(response['successful']) @httpretty.activate def test_login_user(self): """ Test login user api: method = get resource = <user> data = password=<password> """ data = (USR, PW) url = self.files_url content = {'file1': 'foo.txt', 'file2': 'dir/foo.txt'} content_jsoned = json.dumps(content) httpretty.register_uri(httpretty.GET, url, status=200, body=content_jsoned) response = self.cm.do_login(data) self.assertIn('content', response) self.assertIsInstance(response['content'], str) self.assertTrue(response['successful']) @httpretty.activate def test_login_user_failed(self): """ Test login user api with weak password: method = GET resource = <user> data = password=<password> """ data = ('bad_user', 'bad_pass') url = self.files_url httpretty.register_uri(httpretty.GET, url, status=401) response = self.cm.do_login(data) self.assertIn('content', response) self.assertIsInstance(response['content'], str) self.assertFalse(response['successful']) @httpretty.activate def test_post_recover_password_not_found(self): """ Test that if /users/<email>/reset POST == 404 then cm return None """ # An unknown user (neither registered nor pending) is a resource not found for the server... email = '*****@*****.**' url = self.user_url + email + '/reset' # ...so the server should return a 404: httpretty.register_uri(httpretty.POST, url, status=404) # and the command manager must return None in this case response = self.cm.do_reqrecoverpass(email) self.assertIsNone(response) @httpretty.activate def test_post_recover_password_accept(self): """ Test that if /users/<email>/reset POST == 202 then cm return True """ email = '*****@*****.**' url = self.user_url + email + '/reset' httpretty.register_uri(httpretty.POST, url, status=202) response = self.cm.do_reqrecoverpass(email) self.assertTrue(response) @httpretty.activate def test_put_recover_password_not_found(self): """ Test that if /users/<email> PUT == 404 then cm return None """ email = '*****@*****.**' recoverpass_code = os.urandom(16).encode('hex') new_password = '******' url = self.user_url + email httpretty.register_uri(httpretty.PUT, url, status=404) data = email, recoverpass_code, new_password response = self.cm.do_recoverpass(data) self.assertFalse(response) @httpretty.activate def test_put_recover_password_ok(self): """ Test that if /users/<email> PUT == 200 then cm return True """ email = '*****@*****.**' recoverpass_code = os.urandom(16).encode('hex') new_password = '******' url = self.user_url + email httpretty.register_uri(httpretty.PUT, url, status=200) data = email, recoverpass_code, new_password response = self.cm.do_recoverpass(data) self.assertTrue(response) # files: @httpretty.activate def test_download_normal_file(self): url = ''.join((self.files_url, 'file.txt')) httpretty.register_uri(httpretty.GET, url, status=201) data = {'filepath': 'file.txt'} response = self.cm.do_download(data) self.assertEqual(response['successful'], True) @httpretty.activate def test_download_file_not_exists(self): url = ''.join((self.files_url, 'file.tx')) httpretty.register_uri(httpretty.GET, url, status=404) data = {'filepath': 'file.tx'} response = self.cm.do_download(data) self.assertEqual(response['successful'], False) self.assertIsInstance(response['content'], str) @httpretty.activate def test_do_upload_success(self): # prepare fake server url = ''.join((self.files_url, 'foo.txt')) msg = {'server_timestamp': time.time()} js = json.dumps(msg) httpretty.register_uri(httpretty.POST, url, status=201, body=js, content_type="application/json") # call api response = self.cm.do_upload({'filepath': 'foo.txt', 'md5': 'test_md5'}) self.assertTrue(response['successful']) self.assertEqual(response['content'], msg) @httpretty.activate def test_encode_of_url_with_strange_char(self): """ Test the url encode of filename with strange char. I use upload method for example and i expect that httpretty answer at the right URL. """ # Create the file with strange name strange_filename = 'name%with#strange~char' strange_filepath = os.path.join(TEST_SHARING_FOLDER, strange_filename) with open(strange_filepath, 'w') as f: f.write('file with strange name content') # prepare fake server encoded_filename = urllib.quote(strange_filename, self.cm.ENCODER_FILTER) url = ''.join((self.files_url, encoded_filename)) print url msg = {'server_timestamp': time.time()} js = json.dumps(msg) httpretty.register_uri(httpretty.POST, url, status=201, body=js, content_type="application/json") # call api response = self.cm.do_upload({'filepath': strange_filename, 'md5': 'test_md5'}) self.assertTrue(response['successful']) self.assertEqual(response['content'], msg) # actions: @httpretty.activate def test_do_move(self): url = ''.join((self.actions_url, 'move')) msg = {'server_timestamp': time.time()} js = json.dumps(msg) httpretty.register_uri(httpretty.POST, url, status=201, body=js, content_type="application/json") response = self.cm.do_move({'src': 'foo.txt', 'dst': 'folder/foo.txt'}) self.assertTrue(response['successful']) self.assertEqual(response['content'], msg) @httpretty.activate def test_do_delete(self): url = ''.join((self.actions_url, 'delete')) msg = {'server_timestamp': time.time()} js = json.dumps(msg) httpretty.register_uri(httpretty.POST, url, status=201, body=js, content_type="application/json") d = {'filepath': 'foo.txt'} response = self.cm.do_delete(d) self.assertTrue(response['successful']) self.assertEqual(response['content'], msg) @httpretty.activate def test_do_modify(self): url = ''.join((self.files_url, 'foo.txt')) msg = {'server_timestamp': time.time()} js = json.dumps(msg) httpretty.register_uri(httpretty.PUT, url, status=201, body=js, content_type="application/json") response = self.cm.do_modify({'filepath': 'foo.txt', 'md5': 'test_md5'}) self.assertTrue(response['successful']) self.assertEqual(response['content'], msg) @httpretty.activate def test_do_copy(self): url = ''.join([self.actions_url, 'copy']) d = {'src': 'foo.txt', 'dst': 'folder/foo.txt'} msg = {'server_timestamp': time.time()} js = json.dumps(msg) httpretty.register_uri(httpretty.POST, url, status=201, body=js, content_type="application/json") response = self.cm.do_copy(d) self.assertTrue(response['successful']) self.assertEqual(response['content'], msg) @httpretty.activate def test_get_server_snapshot(self): url = self.files_url msg = {'files': 'foo.txt'} js = json.dumps(msg) httpretty.register_uri(httpretty.GET, url, status=201, body=js, content_type="application/json") response = self.cm.do_get_server_snapshot('') self.assertTrue(response['successful']) self.assertEqual(response['content'], msg)
class Master: def msg_callback(self, ch, method, properties, body): callback_set = {'SUCCESS': self.success, 'FAIL': self.fail, 'AWAKE': self.update_slave_response_time, 'STOP': self.stop, 'ADD_SLAVE': self.add_slave, 'KILL_SLAVE': self.kill_slave, 'RESTART_SLAVE': self.restart_slave, 'STAT': self.stat, 'START': self.start, 'RECONFIGURE': self.configure, 'REFRESH': self.refresh } try: command = body[:body.find(' ')] info = body[body.find(' ')+1:] if command in callback_set: callback_set[command](ujson.loads(info)) else: logging.debug(" [x] Unknown command %r" % (str(body),)) except KeyError as e: if str(e) == "'Queue.DeclareOk'": logging.debug("Queue.DelcareOk at %r" % (str(body),)) else: logging.error("Unknown KeyError at %r:" % (str(body),)) except RuntimeError as e: if 'recursion' in str(e): logging.error('MAXIMUM RECURSION ERROR') def __init__(self, conf_file): self.config = ConfigParser.ConfigParser(allow_no_value=True) self.clean_time_gap = None self.wait_time_for_slave = None self.master_queue_name = None self.task_queue_name = None self.task_queue_size_limit = None self.task_file_name = None self.task_counter_file = None self.ssh_key = None self.s3_bucket = None self.s3_folder = None self.slave_num_every_packup = None self.slave_max_sec_each_task = None self.slave_python_version = None self.master_ip = None self.slaves_ip = None self.slave_awake_frequency = None self.configure(conf_file) self.last_wake_time = None self.repeated_timer = None self.is_started = False self.pop_forever_handler = None logging.info('Starting task manager...') self.task_manager = TaskManager(self.task_file_name, self.task_counter_file) logging.info('Starting slave manager...') self.slave_manager = SlaveManager(master_ip=self.master_ip, slaves_ip=self.slaves_ip, ssh_key=self.ssh_key, s3_bucket=self.s3_bucket, s3_folder=self.s3_folder, slave_num_every_packup=self.slave_num_every_packup, slave_max_sec_each_task=self.slave_max_sec_each_task, slave_python_version=self.slave_python_version, slave_awake_frequency=self.slave_awake_frequency, slave_buffer_size=1) logging.info('Starting connection manager...') self.message_connection = ConnectionManager(queue_name=self.master_queue_name, durable=False, callback=self.msg_callback, no_ack=True) def run(self): logging.info(' [*] Waiting for messages. To exit press CTRL+C') try: self.message_connection.start_accepting_message() except KeyboardInterrupt: logging.info('Stopping master...') master.stop(None) except EOFError: logging.info('Download finishes. Shutting down master.') master.stop(None) # except Exception as e: # logging.info(str(e)) # logging.info('Stopping master...') # TODO: write all configuration in one file def configure(self, conf_file): self.config.read(conf_file) self.clean_time_gap = self.config.getint('main', 'clean_time_gap') self.wait_time_for_slave = self.config.getint('main', 'wait_time_for_slave') self.slave_awake_frequency = self.config.get('main', 'slave_awake_frequency') self.master_ip = self.config.get('main', 'master_private_ip') self.slaves_ip = self.config.get('main', 'slaves_private_ip') self.master_queue_name = self.config.get('main', 'master_queue_name') self.task_queue_name = self.config.get('main', 'task_queue_name') self.task_file_name = self.config.get('main', 'task_file') self.task_queue_size_limit = int(self.config.get('main', 'task_queue_size_limit')) self.task_counter_file = self.config.get('main', 'task_counter_file') self.ssh_key = self.config.get('main', 'ssh_key') self.s3_bucket = self.config.get('main', 's3_bucket') self.s3_folder = self.config.get('main', 's3_folder') self.slave_num_every_packup = self.config.get('main', 'slave_num_every_packup') self.slave_max_sec_each_task = self.config.get('main', 'slave_max_sec_each_task') self.slave_python_version = self.config.get('main', 'slave_python_version') def add_slave(self, slave_info): if self.slave_manager.exist_slave(slave_info): logging.info('Slave ' + slave_info['host'] + ' already exists.') return logging.info('master: add slave' + str(slave_info)) new_slave_info = self.slave_manager.add_slave(slave_info) self.slave_manager.run_slave(new_slave_info) # TODO: def kill_slave(self, slave_info): if not self.slave_manager.exist_slave(slave_info): return logging.info('kill slave ' + str(slave_info)) self.slave_manager.kill_slave(slave_info) def restart_slave(self, slave_info): logging.info(slave_info['host']) logging.info('restart_slave' + str(slave_info)) self.kill_slave(slave_info) self.add_slave(slave_info) def start(self, info): logging.info('Master Starts') self.last_wake_time = datetime.datetime.utcnow() self.is_started = True self.pop_forever_handler = threading.Thread(target=self.start_popping_tasks) self.pop_forever_handler.start() self.repeated_timer = RepeatedTimer(self.clean_time_gap, self.notice_refresh, None) def pop_forever(self): self.start_popping_tasks() def get_task_queue_size(self): pass # TODO: There is a bottle neck here def start_popping_tasks(self): task_connection = ConnectionManager(queue_name=self.task_queue_name, durable=True, no_ack=False) eof_reached = False while self.is_started and not eof_reached: current_task_queue_size = task_connection.get_task_queue_size() while self.is_started and current_task_queue_size < self.task_queue_size_limit: task = self.task_manager.pop_task() if task is None: # TODO: Don't use Error. Just break and handle the case later in this function logging.info('EOF Reached') eof_reached = True break message = 'WORK ' + ujson.dumps(task) task_connection.publish(message) current_task_queue_size += 1 task_connection.stop() def fail(self, slave_task_info): self.task_manager.add_task(slave_task_info['task']) self.slave_manager.update_last_response(slave_task_info) def success(self, slave_task_info): slave_info = self.slave_manager.update_last_response(slave_task_info) def update_slave_response_time(self, slave_task_info): slave_info = self.slave_manager.update_last_response(slave_task_info) def stop(self, info): self.is_started = False self.notice_slaves_stop() if self.pop_forever_handler is not None: self.pop_forever_handler.join() if self.repeated_timer is not None: self.repeated_timer.stop() self.slave_manager.stop() self.task_manager.stop() self.message_connection.stop() def notice_slaves_stop(self): task_connection = ConnectionManager(queue_name=self.task_queue_name, durable=True, no_ack=False) screen_list = [key for key in self.slave_manager.slave_dict.keys()] for screen in screen_list: task_connection.publish('STOP {}') # task_connection.broadcast_task('STOP {}') task_connection.stop() def refresh(self, info): cur_progress, total_task = self.task_manager.get_progress() logging.info('downloading {}/{} files'.format(cur_progress, total_task)) if not self.is_started: return # if time interval met, check failed slave if self.last_wake_time is None: self.last_wake_time = datetime.datetime.utcnow() if self.last_wake_time + datetime.timedelta( seconds=self.clean_time_gap) > datetime.datetime.utcnow(): return failed_slaves = self.slave_manager.get_failed_slaves(self.wait_time_for_slave) if len(failed_slaves) != 0: logging.info('Finding failed slaves... ' + str(failed_slaves)) for slave in failed_slaves: self.restart_slave(slave) self.last_wake_time = datetime.datetime.utcnow() def notice_refresh(self, info): try: self.message_connection.publish('REFRESH {}') except IndexError: logging.critical('INDEX_ERROR') def stat(self, info): logging.info('=====================================') logging.info('Num of slave: ', self.slave_manager.get_num_slaves()) logging.info('=====================================') if len(info) > 0: for slave in self.slave_manager.slave_list: if slave['last_response'] is None: delta = 'new slave' else: delta = datetime.datetime.utcnow() - slave['last_response'] logging.info(slave['host'], '|', slave['queue'], '|', delta) logging.info('====================================')