def Test_get_products(self): product = querydb.get_products(activated=True) logger.info("Active products: %s", product) for row in product: print row product = querydb.get_products(activated=False) logger.info("Non active products: %s", product) for row in product: print row product = querydb.get_products(masked=False) logger.info("Not masked products: %s", product) for row in product: print row self.assertEqual(1, 1)
def Test_get_products(self): product = querydb.get_products(activated=True) logger.info("Active products: %s", product) for row in product: print row product = querydb.get_products(activated=False) logger.info("Non active products: %s", product) for row in product: print row product = querydb.get_products(masked=False) logger.info("Not masked products: %s", product) for row in product: print row self.assertEqual(1, 1)
def test_all_products_to_json(self): def row2dict(row): d = {} for column_name in list(row.c.keys()): # all_cols: d[column_name] = str(getattr(row, column_name)) return d # get full distinct list of products (native only) db_products = querydb.get_products() try: db_products.__len__() except AttributeError: db_products = querydb.get_product_native(allrecs=True) self.assertTrue(db_products.__len__() > 0) products_dict_all = [] # loop the products list for product in db_products: if python_version == 2: prod_dict = row2dict(product) if python_version == 3: prod_dict = product productcode = prod_dict['productcode'] version = prod_dict['version'] p = Product(product_code=productcode, version=version) # does the product have mapsets AND subproducts? all_prod_mapsets = p.mapsets all_prod_subproducts = p.subproducts if all_prod_mapsets.__len__() > 0 and all_prod_subproducts.__len__() > 0: prod_dict['productmapsets'] = [] for mapset in all_prod_mapsets: mapset_info = querydb.get_mapset(mapsetcode=mapset, allrecs=False) mapset_dict = row2dict(mapset_info) mapset_dict['mapsetdatasets'] = [] all_mapset_datasets = p.get_subproducts(mapset=mapset) for subproductcode in all_mapset_datasets: dataset_info = querydb.get_subproduct(productcode=productcode, version=version, subproductcode=subproductcode) dataset_dict = row2dict(dataset_info) dataset = p.get_dataset(mapset=mapset, sub_product_code=subproductcode) completeness = dataset.get_dataset_normalized_info() dataset_dict['datasetcompleteness'] = completeness mapset_dict['mapsetdatasets'].append(dataset_dict) prod_dict['productmapsets'].append(mapset_dict) products_dict_all.append(prod_dict) # See ES2-596 self.assertEqual(len(db_products), len(products_dict_all))
def test_all_products_to_json(self): def row2dict(row): d = {} for column_name in row.c.keys(): # all_cols: d[column_name] = str(getattr(row, column_name)) return d # get full distinct list of products (native only) db_products = querydb.get_products(echo=False) try: db_products.__len__() except AttributeError: db_products = querydb.get_product_native(allrecs=True, echo=False) self.assertTrue(db_products.__len__() > 0) products_dict_all = [] # loop the products list for product in db_products: prod_dict = row2dict(product) productcode = prod_dict['productcode'] version = prod_dict['version'] p = Product(product_code=productcode, version=version) # does the product have mapsets AND subproducts? all_prod_mapsets = p.mapsets all_prod_subproducts = p.subproducts if all_prod_mapsets.__len__() > 0 and all_prod_subproducts.__len__() > 0: prod_dict['productmapsets'] = [] for mapset in all_prod_mapsets: mapset_info = querydb.get_mapset(mapsetcode=mapset, allrecs=False, echo=False) mapset_dict = row2dict(mapset_info) mapset_dict['mapsetdatasets'] = [] all_mapset_datasets = p.get_subproducts(mapset=mapset) for subproductcode in all_mapset_datasets: dataset_info = querydb.get_subproduct(productcode=productcode, version=version, subproductcode=subproductcode, echo=False) dataset_dict = row2dict(dataset_info) dataset = p.get_dataset(mapset=mapset, sub_product_code=subproductcode) completeness = dataset.get_dataset_normalized_info() dataset_dict['datasetcompleteness'] = completeness mapset_dict['mapsetdatasets'].append(dataset_dict) prod_dict['productmapsets'].append(mapset_dict) products_dict_all.append(prod_dict) self.assertEquals(len(db_products), 31)
def test_data_management(self): import json db_products = querydb.get_products(activated=True) if db_products.__len__() > 0: products_dict_all = [] # loop the products list for row in db_products: prod_dict = functions.row2dict(row) productcode = prod_dict['productcode'] version = prod_dict['version'] p = Product(product_code=productcode, version=version) # print productcode # does the product have mapsets AND subproducts? all_prod_mapsets = p.mapsets all_prod_subproducts = p.subproducts if all_prod_mapsets.__len__( ) > 0 and all_prod_subproducts.__len__() > 0: prod_dict['productmapsets'] = [] for mapset in all_prod_mapsets: mapset_dict = [] # print mapset mapset_info = querydb.get_mapset(mapsetcode=mapset, allrecs=False) # if mapset_info.__len__() > 0: mapset_dict = functions.row2dict(mapset_info) # else: # mapset_dict['mapsetcode'] = mapset mapset_dict['mapsetdatasets'] = [] all_mapset_datasets = p.get_subproducts(mapset=mapset) for subproductcode in all_mapset_datasets: # print 'productcode: ' + productcode # print 'version: ' + version # print 'subproductcode: ' + subproductcode dataset_info = querydb.get_subproduct( productcode=productcode, version=version, subproductcode=subproductcode) # print dataset_info # dataset_info = querydb.db.product.get(productcode, version, subproductcode) # dataset_dict = {} if dataset_info is not None: dataset_dict = functions.row2dict(dataset_info) # dataset_dict = dataset_info.__dict__ # del dataset_dict['_labels'] if hasattr(dataset_info, 'frequency_id'): if dataset_info.frequency_id == 'e15minute' or dataset_info.frequency_id == 'e30minute': dataset_dict[ 'nodisplay'] = 'no_minutes_display' # To be implemented in dataset.py elif dataset_info.frequency_id == 'e1year': dataset_dict[ 'nodisplay'] = 'no_minutes_display' else: dataset = p.get_dataset( mapset=mapset, sub_product_code=subproductcode) completeness = dataset.get_dataset_normalized_info( ) dataset_dict[ 'datasetcompleteness'] = completeness dataset_dict['nodisplay'] = 'false' dataset_dict['mapsetcode'] = mapset_dict[ 'mapsetcode'] dataset_dict[ 'mapset_descriptive_name'] = mapset_dict[ 'descriptive_name'] mapset_dict['mapsetdatasets'].append( dataset_dict) else: pass prod_dict['productmapsets'].append(mapset_dict) products_dict_all.append(prod_dict) prod_json = json.dumps(products_dict_all, ensure_ascii=False, sort_keys=True, indent=4, separators=(', ', ': ')) datamanagement_json = '{"success":"true", "total":'\ + str(db_products.__len__())\ + ',"products":'+prod_json+'}' else: datamanagement_json = '{"success":false, "error":"No data sets defined!"}'
def push_data_ftp(dry_run=False, user=None, psw=None, url=None, trg_dir=None, masked=True): # Synchronized data towards an ftp server (only for JRC) # It replaces, since the new srv-ies-ftp.jrc.it ftp is set, the bash script: mirror_to_ftp.sh # Configuration: it looks at all 'non-masked' products and pushes them # For the mapsets, find what is in the filesystem, and pushes only the 'largest' # It uses a command like: # lftp -e "mirror -RLe /data/processing/vgt-ndvi/sv2-pv2.1/SPOTV-Africa-1km/derived/10dmax-linearx2/ # /narma/eStation_2.0/processing/vgt-ndvi/sv2-pv2.1/SPOTV-Africa-1km/derived/10dmax-linearx2/;exit" # -u narma:JRCVRw2960 sftp://srv-ies-ftp.jrc.it"" >> /eStation2/log/push_data_ftp.log # spec_logger = log.my_logger('apps.es2system.push_data_ftp') try: from config import server_ftp except: logger.warning('Configuration file for ftp sync not found. Exit') return 1 if user is None: user = server_ftp.server['user'] if psw is None: psw = server_ftp.server['psw'] if url is None: url = server_ftp.server['url'] if trg_dir is None: trg_dir = server_ftp.server['data_dir'] # Create an ad-hoc file for the lftp command output (beside the standard logger) logfile = es_constants.es2globals['log_dir'] + 'push_data_ftp.log' message = time.strftime( "%Y-%m-%d %H:%M") + ' INFO: Running the ftp sync now ... \n' logger.debug("Entering routine %s" % 'push_data_ftp') # Loop over 'not-masked' products products = querydb.get_products(masked=False) # produts=products[21:23] # test a subset for row in products: prod_dict = functions.row2dict(row) productcode = prod_dict['productcode'] version = prod_dict['version'] spec_logger.info('Working on product {}/{}'.format( productcode, version)) # TEMP - For testing only # if productcode!='vgt-ndvi' or version !='sv2-pv2.2': # continue # Check it if is in the list of 'exclusions' defined in ./config/server_ftp.py key = '{}/{}'.format(productcode, version) skip = False if key in server_ftp.exclusions: skip = True logger.debug('Do not sync for {}/{}'.format(productcode, version)) p = Product(product_code=productcode, version=version) all_prod_mapsets = p.mapsets all_prod_subproducts = p.subproducts # Check there is at least one mapset and one subproduct if all_prod_mapsets.__len__() > 0 and all_prod_subproducts.__len__( ) > 0 and not skip: # In case of several mapsets, check if there is a 'larger' one if len(all_prod_mapsets) > 1: mapset_to_use = [] for my_mapset in all_prod_mapsets: mapset_info = querydb.get_mapset(mapsetcode=my_mapset, allrecs=False) if hasattr(mapset_info, "mapsetcode"): my_mapobj = MapSet() my_mapobj.assigndb(my_mapset) larger_mapset = my_mapobj.get_larger_mapset() if larger_mapset is not None: if larger_mapset not in mapset_to_use: mapset_to_use.append(larger_mapset) else: if my_mapset not in mapset_to_use: mapset_to_use.append(my_mapset) else: mapset_to_use = all_prod_mapsets # Loop over existing mapset for mapset in mapset_to_use: all_mapset_datasets = p.get_subproducts(mapset=mapset) # Loop over existing subproducts for subproductcode in all_mapset_datasets: # Get info - and ONLY for NOT masked products dataset_info = querydb.get_subproduct( productcode=productcode, version=version, subproductcode=subproductcode, masked=masked) # -> TRUE means only NOT masked sprods if dataset_info is not None: dataset_dict = functions.row2dict(dataset_info) dataset_dict['mapsetcode'] = mapset logger.debug('Working on {}/{}/{}/{}'.format( productcode, version, mapset, subproductcode)) subdir = functions.set_path_sub_directory( productcode, subproductcode, dataset_dict['product_type'], version, mapset) source = data_dir + subdir target = trg_dir + subdir # command = 'lftp -e "mirror -RLe {} {};exit" -u {}:{} {}"" >> {}'.format(source,target,user,psw,url,logfile) command = 'lftp -e "mirror -RLe {} {};exit" -u {}:{} {}"" >> /dev/null'.format( source, target, user, psw, url) logger.debug("Executing %s" % command) spec_logger.info( 'Working on mapset/subproduct {}/{} \n'.format( mapset, subproductcode)) # return try: status = os.system(command) if status: logger.error("Error in executing %s" % command) spec_logger.error("Error in executing %s" % command) except: logger.error( 'Error in executing command: {}'.format( command)) spec_logger.error( 'Error in executing command: {}'.format( command))