def run(self, source, outbox_path):
""" This method runs the *process* method of dynamic shared library containing the plugin.
:param source: The path in file system where is located the downloaded file.
:type source: String
:param outbox_path: The path of the processed products folder
:type source: String
:returns: None
:rtype: None
"""
try:
product_name = source.split('/')[-2]
destination = outbox_path + product_name + "-" + self.plugin_name + '/'
if not os.path.exists(destination):
os.makedirs(destination)
cdll.LoadLibrary(self.plugin_path)
libc = CDLL(self.plugin_path)
product_path = source.encode('utf-8')
destination_path = destination.encode('utf-8')
libc.process.argtypes = [c_char_p]
libc.process(product_path, destination_path)
os.dup2(self.stdout, 1)
self.read_pipe()
except Exception as err:
logger.warn("Failed scientific-processor plugin with name: " +
self.plugin_name)
logger.debug(
"(SystemManager set_first_installation_config) Unexpected error:"
)
logger.debug(err)
return False
def start(self):
""" Download event handler. The entry point of the *Downloader* module.
"""
logger.debug("(Downloader start) ")
self.pending_tasks += 1
if self.downloading:
return
self.downloading = True
for tile in self.configuration.tiles:
search_filter = self.create_search_filter(tile)
products_list = self.datasource.get_products_list(search_filter)
for pending_product in products_list:
self.productService.add_new_product(
Product(name=str(pending_product),
status=ProductStatus.pending))
products = [
self.queue.put(product)
for product in self.productService.get_pending_products()
]
while self.pending_tasks:
for i in range(int(self.configuration.parallel_downloads)):
t = DownloaderJob(self.queue)
t.daemon = True
t.start()
self.pending_tasks -= 1
self.downloading = False
def start_processing(self):
""" The entry point of the *ProcessingPipeManager* class.
"""
logger.debug("(ProcessingPipeManager start_processing) ")
ps = ProductsService()
for product in ps.get_products_to_process():
self.process_product(product.name)
def refresh_configurations(self):
""" This method reload the configurations from the *Downloader* config file.
"""
logger.debug("(Downloader refresh_configurations)")
self.configurationManager.load_configuration()
self.configuration = self.configurationManager.configuration
logger.debug("(Downloader refresh_configurations) finished")
def __init__(self, configurations):
logger.debug("(Datasource __init__)")
self.configurations = configurations
self.abm = AmazonBucketManager(configurations)
self.productDownloader = ProductDownloader(configurations.inbox_path,
configurations.files,
configurations.aws_domain)
return
def __init__(self):
logger.debug("(ConfigurationManager __init__)")
self.start_date = None
self.end_date = None
self.tiles = []
self.files = []
self.inbox_path = None
self.parallel_downloads = 0
def __init__(self, queue):
logger.debug("(DownloaderJob __init__)")
self.configurationManager = ConfigurationManager()
self.configuration = self.configurationManager.get_configuration()
self.datasource = Datasource(self.configuration)
self.productService = ProductsService()
self.queue = queue
threading.Thread.__init__(self)
def __init__(self):
logger.debug("(Downloader __init__)")
self.productService = ProductsService()
self.downloading = False
self.configurationManager = ConfigurationManager()
self.configuration = self.configurationManager.get_configuration()
self.queue = queue.Queue()
self.pending_tasks = 0
self.datasource = Datasource(self.configuration)
def __init__(self):
""" This class manages configurations, it extracts the configurations strings from the
config file and put the values in the *Configuration* class.
"""
logger.debug("(ConfigurationManager __init__)")
self.config = configparser.ConfigParser()
self.config.read("/usr/downloader/src/core/config/config.cfg")
self.configuration = Configuration()
self.load_configuration()
def create_database(self):
try:
db = DB()
db.create_db()
return True
except Exception as err:
logger.debug("(SystemManager create_database) Unexpected error:")
logger.debug(err)
return False
def process_product(self, product_name):
""" This method is the wrapper of the web request to trigger a new processing task.
:param product_name: The name of the product to process
:type product_name: String
"""
logger.debug("(ProcessingPipeManager process_product) ")
product_path = self.config['FOLDERS']['inbox_path'] + product_name.replace("/", "-")[:-1]
self.notify_to_scientific_processor(product_path + '/')
def load_configuration(self):
logger.debug("(ConfigurationManager load_configuration)")
self.configuration.start_date = self.load_start_date()
self.configuration.end_date = self.load_end_date()
self.configuration.tiles = self.load_tiles()
self.configuration.files = self.load_files_to_download()
self.configuration.aws_xmlns = self.load_aws_xmlns()
self.configuration.aws_products_regex = self.load_aws_products_regex()
self.configuration.aws_domain = self.load_aws_domain()
self.configuration.inbox_path = self.load_inbox_path()
self.configuration.parallel_downloads = self.load_parallel_downloads()
logger.debug("(ConfigurationManager load_configuration) finished")
def trigger_downloader(self):
logger.debug("(SystemManager trigger_downloader) ")
logger.debug("(SystemManager trigger_downloader) call downloader ")
try:
self.downloader.start()
except Exception as err:
logger.debug("(SystemManager trigger_downloader) Unexpeted error:")
logger.debug(err)
def set_first_installation_config(self, new_status):
try:
value = 'true' if new_status else 'false'
config = configparser.RawConfigParser()
config.read(self.config_file_path)
config.set('SYSTEM_STATUS', 'first_installation', value)
with open(self.config_file_path, 'w') as configfile:
config.write(configfile)
return True
except Exception as err:
logger.debug(
"(DownloaderManager set_first_installation_config) Unexpected error:"
)
logger.debug(err)
return False
def notify_to_scientific_processor(self, file_path):
""" This method launches the http request to trigger a new processing process.
It represent the networking interface to the scientific processor.
:param file_path: The path where is located the product to process
:type file_path: String
"""
logger.debug("(ProcessingPipeManager notify_to_scientific_processor) ")
body = {"path": file_path}
params = json.dumps(body).encode('utf8')
req = urllib.request.Request("http://localhost:5002/process",
data=params,
headers={'content-type': 'application/json'})
response = urllib.request.urlopen(req)
def load_products(self, tile, year, paginated=False):
""" *load_products* load list products form the Amazon *AWS* Bucket of Sinergise *Sentinel-2 on AWS*.
This method contains the *AWS* APIs semantic it is the core of Adapter.
It manages also pagination.
:param tile: Tile name
:type tile: String
:param year: Year of interest, the method extract the whole list of products for this year
:type year: String
:param paginated: if this parameter is *True* that means that the list provided from AWS is
paginated and the next page starts after the *last_item*
:type year: Boolean
:returns: None
:rtype: None
This method stores the list of products in the classes property *product_list*.
"""
logger.debug("(AmazonBucketManager load_products)")
# Generate url for year
url = self.generate_url(tile, year)
# Append to url start_from attribute if the request is paginated
if paginated:
url = url + "&start-after=" + self.last_item
# Http request
response = urllib.request.urlopen(url)
root = ET.fromstring(response.read().decode('utf-8'))
# Extract data from xml
contents = root.findall('{' + self.config.aws_xmlns + '}Contents')
for item in contents:
key = item.find('{' + self.config.aws_xmlns + '}Key').text
product_path = ''
try:
product_path = re.search(self.config.aws_products_regex, key)
except ValueError:
logger.warn("Product not found for key: " + key)
self.product_list.append(product_path.group(0))
# Check if the page is truncated (paginated case)
if root.find('{' + self.config.aws_xmlns +
'}IsTruncated').text == 'true':
self.last_item = contents[-1].find('{' + self.config.aws_xmlns +
'}Key').text
return True
else:
return False
def get_products_list(self, searchFilter):
""" This method manages the retrieval of aws products.
This method contains the *AWS* APIs semantic it is the core of Adapter.
It manages also pagination.
:param searchFilter: Search parameters to filter the available files in the bucket
:type searchFilter: SearchFilter
:returns: The pending products available in the Amazon bucket.
:rtype: None
.. todo:: Optimize list retrieval using set end date year instead *NOW* year.
"""
logger.debug("(AmazonBucketManager get_products_list)")
tile = searchFilter.tile
self.product_list = []
self.last_item = None
year = int(searchFilter.start_date.year)
current_year = datetime.datetime.now().year
pending_products = []
# Extract whole list of products via amazon, from start year to now
# TODO: OPTMIZE IT USING THE END DATE FROM CONFIGURATION AND NOT *NOW*
while year <= current_year:
is_truncated = self.load_products(tile, year, False)
while is_truncated:
is_truncated = self.load_products(tile, year, True)
year += 1
# Clean list of products
self.product_list = set(self.product_list)
# Generate dictionary
dict = {}
for product in self.product_list:
date = self.extract_date(str(product))
product_string = str(product)
dict[date] = product_string
# Sort dictionary
dict = OrderedDict(sorted(dict.items()))
# Filter products via date interval
for product_date in dict:
if product_date >= searchFilter.start_date and product_date <= searchFilter.end_date:
pending_products.append(dict[product_date])
return pending_products
def compile_plugins(self):
""" This method compiles every c++ plugin present in the *plugins* folder of *Processor* module
"""
logger.debug("(PluginManager:compile_plugins) start")
dirnames = os.listdir(self.plugins_path)
r = re.compile('^[^\.]')
dirnames = filter(r.match, dirnames)
for plugin_name in dirnames:
try:
completed_without_error = True
p = subprocess.Popen([
"/bin/bash",
"src/core/system_manager/bash/compile-plugins.sh",
self.plugins_path, plugin_name, "var=11; ignore all"
],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
for line in p.stdout.read().decode('utf-8').split("\n"):
if (len(line) > 0):
logging.debug("(BASH - compile-plugins.sh): " + line)
for line in p.stderr.read().decode('utf-8').split("\n"):
if (len(line) > 0):
logging.debug("[ERROR] (BASH compile-plugins.sh): " +
line)
completed_without_error = False
if (completed_without_error):
logging.debug("(ichnosat-manager): Completed compile " +
plugin_name + " plugin")
else:
logging.debug(
"[ERROR] (ichnosat-manager): Failed compilation of scientific_processor plugin '"
+ plugin_name + "'")
except ValueError:
logging.debug(
"[ERROR] (ichnosat-manager): Failed compilation of scientific_processor plugin '"
+ plugin_name + "'")
logging.debug(
"(ichnosat-manager): COMPLETED compile scientific_processor plugins"
)
return "Done"
def download_product(self, product_name):
""" This method launches the http request to download the product via product name.
:param product_name: The name of product to download
:type product_name: String
:returns: None
:rtype: None
"""
logger.debug("(ProductDownloader download_product) ")
logger.debug("(ProductDownloader download_product) product.name: " +
product_name)
new_product_path = self.inbox_path + product_name.replace("/",
"-")[:-1]
if not os.path.exists(new_product_path):
os.makedirs(new_product_path)
files_to_download = self.files_to_download #.split(',')
for file_name in files_to_download:
url = self.domain + product_name + file_name
new_file_path = new_product_path + '/' + file_name
urllib.request.urlretrieve(url, new_file_path)
def get_products_list(self, searchFilter):
""" This method generates the product list from query filters defined in the config file
by the user.
:param searchFilter: Search parameters to filter the available files in the bucket
:type searchFilter: SearchFilter
:returns: The list of available products.
:rtype: List
"""
logger.debug("(Datasource get_products_list)")
products_list = self.abm.get_products_list(searchFilter)
logger.debug("(Datasrouce get_products_list) list of products:")
for product_name in products_list:
logger.debug("(Datasource get_products_list) product_name: " +
product_name)
return products_list
def __init__(self, inbox_path, files_to_download, domain):
logger.debug("(ProductDownloader __init__) ")
self.inbox_path = inbox_path
self.files_to_download = files_to_download
self.domain = domain
return
def get_pending_products(self):
logger.debug("(SystemManager get_pending_products) ")
return self.productService.get_pending_products()
def __init__(self):
logger.debug("(ProcessingPipeManager __init__) ")
self.config = configparser.ConfigParser()
self.config.read("/usr/ichnosat/src/core/processing_pipe/config/config.cfg")
def get_pending_products(self):
logger.debug("(DownloaderManager get_pending_products) ")
return self.productService.get_pending_products()
def set_processing_false(self):
""" This method set the internal status *processing* to False, to accept new requests of processing.
"""
logger.debug("(Producer set_processing_false) ")
self.processing = False
def read_pipe(self):
""" Utility method to manage out stream of dynamic shared libraries.
Reading the out stream pipe.
"""
while self.more_data():
logger.debug(os.read(self.pipe_out, 1024).decode('utf-8'))
