def run(self, source, outbox_path): """ This method runs the *process* method of dynamic shared library containing the plugin. :param source: The path in file system where is located the downloaded file. :type source: String :param outbox_path: The path of the processed products folder :type source: String :returns: None :rtype: None """ try: product_name = source.split('/')[-2] destination = outbox_path + product_name + "-" + self.plugin_name + '/' if not os.path.exists(destination): os.makedirs(destination) cdll.LoadLibrary(self.plugin_path) libc = CDLL(self.plugin_path) product_path = source.encode('utf-8') destination_path = destination.encode('utf-8') libc.process.argtypes = [c_char_p] libc.process(product_path, destination_path) os.dup2(self.stdout, 1) self.read_pipe() except Exception as err: logger.warn("Failed scientific-processor plugin with name: " + self.plugin_name) logger.debug( "(SystemManager set_first_installation_config) Unexpected error:" ) logger.debug(err) return False
def start(self): """ Download event handler. The entry point of the *Downloader* module. """ logger.debug("(Downloader start) ") self.pending_tasks += 1 if self.downloading: return self.downloading = True for tile in self.configuration.tiles: search_filter = self.create_search_filter(tile) products_list = self.datasource.get_products_list(search_filter) for pending_product in products_list: self.productService.add_new_product( Product(name=str(pending_product), status=ProductStatus.pending)) products = [ self.queue.put(product) for product in self.productService.get_pending_products() ] while self.pending_tasks: for i in range(int(self.configuration.parallel_downloads)): t = DownloaderJob(self.queue) t.daemon = True t.start() self.pending_tasks -= 1 self.downloading = False
def start_processing(self): """ The entry point of the *ProcessingPipeManager* class. """ logger.debug("(ProcessingPipeManager start_processing) ") ps = ProductsService() for product in ps.get_products_to_process(): self.process_product(product.name)
def refresh_configurations(self): """ This method reload the configurations from the *Downloader* config file. """ logger.debug("(Downloader refresh_configurations)") self.configurationManager.load_configuration() self.configuration = self.configurationManager.configuration logger.debug("(Downloader refresh_configurations) finished")
def __init__(self, configurations): logger.debug("(Datasource __init__)") self.configurations = configurations self.abm = AmazonBucketManager(configurations) self.productDownloader = ProductDownloader(configurations.inbox_path, configurations.files, configurations.aws_domain) return
def __init__(self): logger.debug("(ConfigurationManager __init__)") self.start_date = None self.end_date = None self.tiles = [] self.files = [] self.inbox_path = None self.parallel_downloads = 0
def __init__(self, queue): logger.debug("(DownloaderJob __init__)") self.configurationManager = ConfigurationManager() self.configuration = self.configurationManager.get_configuration() self.datasource = Datasource(self.configuration) self.productService = ProductsService() self.queue = queue threading.Thread.__init__(self)
def __init__(self): logger.debug("(Downloader __init__)") self.productService = ProductsService() self.downloading = False self.configurationManager = ConfigurationManager() self.configuration = self.configurationManager.get_configuration() self.queue = queue.Queue() self.pending_tasks = 0 self.datasource = Datasource(self.configuration)
def __init__(self): """ This class manages configurations, it extracts the configurations strings from the config file and put the values in the *Configuration* class. """ logger.debug("(ConfigurationManager __init__)") self.config = configparser.ConfigParser() self.config.read("/usr/downloader/src/core/config/config.cfg") self.configuration = Configuration() self.load_configuration()
def create_database(self): try: db = DB() db.create_db() return True except Exception as err: logger.debug("(SystemManager create_database) Unexpected error:") logger.debug(err) return False
def process_product(self, product_name): """ This method is the wrapper of the web request to trigger a new processing task. :param product_name: The name of the product to process :type product_name: String """ logger.debug("(ProcessingPipeManager process_product) ") product_path = self.config['FOLDERS']['inbox_path'] + product_name.replace("/", "-")[:-1] self.notify_to_scientific_processor(product_path + '/')
def load_configuration(self): logger.debug("(ConfigurationManager load_configuration)") self.configuration.start_date = self.load_start_date() self.configuration.end_date = self.load_end_date() self.configuration.tiles = self.load_tiles() self.configuration.files = self.load_files_to_download() self.configuration.aws_xmlns = self.load_aws_xmlns() self.configuration.aws_products_regex = self.load_aws_products_regex() self.configuration.aws_domain = self.load_aws_domain() self.configuration.inbox_path = self.load_inbox_path() self.configuration.parallel_downloads = self.load_parallel_downloads() logger.debug("(ConfigurationManager load_configuration) finished")
def trigger_downloader(self): logger.debug("(SystemManager trigger_downloader) ") logger.debug("(SystemManager trigger_downloader) call downloader ") try: self.downloader.start() except Exception as err: logger.debug("(SystemManager trigger_downloader) Unexpeted error:") logger.debug(err)
def set_first_installation_config(self, new_status): try: value = 'true' if new_status else 'false' config = configparser.RawConfigParser() config.read(self.config_file_path) config.set('SYSTEM_STATUS', 'first_installation', value) with open(self.config_file_path, 'w') as configfile: config.write(configfile) return True except Exception as err: logger.debug( "(DownloaderManager set_first_installation_config) Unexpected error:" ) logger.debug(err) return False
def notify_to_scientific_processor(self, file_path): """ This method launches the http request to trigger a new processing process. It represent the networking interface to the scientific processor. :param file_path: The path where is located the product to process :type file_path: String """ logger.debug("(ProcessingPipeManager notify_to_scientific_processor) ") body = {"path": file_path} params = json.dumps(body).encode('utf8') req = urllib.request.Request("http://localhost:5002/process", data=params, headers={'content-type': 'application/json'}) response = urllib.request.urlopen(req)
def load_products(self, tile, year, paginated=False): """ *load_products* load list products form the Amazon *AWS* Bucket of Sinergise *Sentinel-2 on AWS*. This method contains the *AWS* APIs semantic it is the core of Adapter. It manages also pagination. :param tile: Tile name :type tile: String :param year: Year of interest, the method extract the whole list of products for this year :type year: String :param paginated: if this parameter is *True* that means that the list provided from AWS is paginated and the next page starts after the *last_item* :type year: Boolean :returns: None :rtype: None This method stores the list of products in the classes property *product_list*. """ logger.debug("(AmazonBucketManager load_products)") # Generate url for year url = self.generate_url(tile, year) # Append to url start_from attribute if the request is paginated if paginated: url = url + "&start-after=" + self.last_item # Http request response = urllib.request.urlopen(url) root = ET.fromstring(response.read().decode('utf-8')) # Extract data from xml contents = root.findall('{' + self.config.aws_xmlns + '}Contents') for item in contents: key = item.find('{' + self.config.aws_xmlns + '}Key').text product_path = '' try: product_path = re.search(self.config.aws_products_regex, key) except ValueError: logger.warn("Product not found for key: " + key) self.product_list.append(product_path.group(0)) # Check if the page is truncated (paginated case) if root.find('{' + self.config.aws_xmlns + '}IsTruncated').text == 'true': self.last_item = contents[-1].find('{' + self.config.aws_xmlns + '}Key').text return True else: return False
def get_products_list(self, searchFilter): """ This method manages the retrieval of aws products. This method contains the *AWS* APIs semantic it is the core of Adapter. It manages also pagination. :param searchFilter: Search parameters to filter the available files in the bucket :type searchFilter: SearchFilter :returns: The pending products available in the Amazon bucket. :rtype: None .. todo:: Optimize list retrieval using set end date year instead *NOW* year. """ logger.debug("(AmazonBucketManager get_products_list)") tile = searchFilter.tile self.product_list = [] self.last_item = None year = int(searchFilter.start_date.year) current_year = datetime.datetime.now().year pending_products = [] # Extract whole list of products via amazon, from start year to now # TODO: OPTMIZE IT USING THE END DATE FROM CONFIGURATION AND NOT *NOW* while year <= current_year: is_truncated = self.load_products(tile, year, False) while is_truncated: is_truncated = self.load_products(tile, year, True) year += 1 # Clean list of products self.product_list = set(self.product_list) # Generate dictionary dict = {} for product in self.product_list: date = self.extract_date(str(product)) product_string = str(product) dict[date] = product_string # Sort dictionary dict = OrderedDict(sorted(dict.items())) # Filter products via date interval for product_date in dict: if product_date >= searchFilter.start_date and product_date <= searchFilter.end_date: pending_products.append(dict[product_date]) return pending_products
def compile_plugins(self): """ This method compiles every c++ plugin present in the *plugins* folder of *Processor* module """ logger.debug("(PluginManager:compile_plugins) start") dirnames = os.listdir(self.plugins_path) r = re.compile('^[^\.]') dirnames = filter(r.match, dirnames) for plugin_name in dirnames: try: completed_without_error = True p = subprocess.Popen([ "/bin/bash", "src/core/system_manager/bash/compile-plugins.sh", self.plugins_path, plugin_name, "var=11; ignore all" ], stdout=subprocess.PIPE, stderr=subprocess.PIPE) for line in p.stdout.read().decode('utf-8').split("\n"): if (len(line) > 0): logging.debug("(BASH - compile-plugins.sh): " + line) for line in p.stderr.read().decode('utf-8').split("\n"): if (len(line) > 0): logging.debug("[ERROR] (BASH compile-plugins.sh): " + line) completed_without_error = False if (completed_without_error): logging.debug("(ichnosat-manager): Completed compile " + plugin_name + " plugin") else: logging.debug( "[ERROR] (ichnosat-manager): Failed compilation of scientific_processor plugin '" + plugin_name + "'") except ValueError: logging.debug( "[ERROR] (ichnosat-manager): Failed compilation of scientific_processor plugin '" + plugin_name + "'") logging.debug( "(ichnosat-manager): COMPLETED compile scientific_processor plugins" ) return "Done"
def download_product(self, product_name): """ This method launches the http request to download the product via product name. :param product_name: The name of product to download :type product_name: String :returns: None :rtype: None """ logger.debug("(ProductDownloader download_product) ") logger.debug("(ProductDownloader download_product) product.name: " + product_name) new_product_path = self.inbox_path + product_name.replace("/", "-")[:-1] if not os.path.exists(new_product_path): os.makedirs(new_product_path) files_to_download = self.files_to_download #.split(',') for file_name in files_to_download: url = self.domain + product_name + file_name new_file_path = new_product_path + '/' + file_name urllib.request.urlretrieve(url, new_file_path)
def get_products_list(self, searchFilter): """ This method generates the product list from query filters defined in the config file by the user. :param searchFilter: Search parameters to filter the available files in the bucket :type searchFilter: SearchFilter :returns: The list of available products. :rtype: List """ logger.debug("(Datasource get_products_list)") products_list = self.abm.get_products_list(searchFilter) logger.debug("(Datasrouce get_products_list) list of products:") for product_name in products_list: logger.debug("(Datasource get_products_list) product_name: " + product_name) return products_list
def __init__(self, inbox_path, files_to_download, domain): logger.debug("(ProductDownloader __init__) ") self.inbox_path = inbox_path self.files_to_download = files_to_download self.domain = domain return
def get_pending_products(self): logger.debug("(SystemManager get_pending_products) ") return self.productService.get_pending_products()
def __init__(self): logger.debug("(ProcessingPipeManager __init__) ") self.config = configparser.ConfigParser() self.config.read("/usr/ichnosat/src/core/processing_pipe/config/config.cfg")
def get_pending_products(self): logger.debug("(DownloaderManager get_pending_products) ") return self.productService.get_pending_products()
def set_processing_false(self): """ This method set the internal status *processing* to False, to accept new requests of processing. """ logger.debug("(Producer set_processing_false) ") self.processing = False
def read_pipe(self): """ Utility method to manage out stream of dynamic shared libraries. Reading the out stream pipe. """ while self.more_data(): logger.debug(os.read(self.pipe_out, 1024).decode('utf-8'))