def Test_get_processingchains_input_products(self):
import json
processingchain_products = querydb.get_processingchains_input_products(
)
if processingchain_products.__len__() > 0:
products_dict_all = []
# loop the products list
for input_product in processingchain_products:
process_id = input_product.process_id
output_mapsetcode = input_product.output_mapsetcode
prod_dict = functions.row2dict(input_product)
# prod_dict = input_product
# del prod_dict['_labels']
prod_dict['productmapsets'] = []
mapset_info = querydb.get_mapset(mapsetcode=output_mapsetcode)
mapset_dict = functions.row2dict(mapset_info)
mapset_dict['mapsetoutputproducts'] = []
output_products = querydb.get_processingchain_output_products(
process_id)
for outputproduct in output_products:
outputproduct_dict = functions.row2dict(outputproduct)
# outputproduct_dict = outputproduct
# del outputproduct_dict['_labels']
mapset_dict['mapsetoutputproducts'].append(
outputproduct_dict)
prod_dict['productmapsets'].append(mapset_dict)
products_dict_all.append(prod_dict)
prod_json = json.dumps(products_dict_all,
ensure_ascii=False,
sort_keys=True,
indent=4,
separators=(', ', ': '))
# logger.info("Processing chains: %s", processingchain_products)
# for row in processingchain_products:
# logger.info("row.dict: %s", row.__dict__)
# logger.info("row.process_id: %s", row.process_id)
# logger.info("row.output_mapsetcode: %s", row.output_mapsetcode)
# logger.info("row.mapsetcode: %s", row.mapsetcode)
# print row.process_id
# print row.output_mapsetcode
# print row.mapsetcode
self.assertEqual(1, 1)
def Test_get_mapset(self):
mapset = querydb.get_mapset(mapsetcode='SPOTV-Africa-1km')
logger.info("Mapset: %s", mapset.pixel_shift_lat)
mapset_dict = functions.row2dict(mapset)
self.assertEqual(1, 1)
def Test_get_processingchains_input_products(self):
import json
processingchain_products = querydb.get_processingchains_input_products()
if processingchain_products.__len__() > 0:
products_dict_all = []
# loop the products list
for input_product in processingchain_products:
process_id = input_product.process_id
output_mapsetcode = input_product.output_mapsetcode
prod_dict = functions.row2dict(input_product)
# prod_dict = input_product
# del prod_dict['_labels']
prod_dict['productmapsets'] = []
mapset_info = querydb.get_mapset(mapsetcode=output_mapsetcode)
mapset_dict = functions.row2dict(mapset_info)
mapset_dict['mapsetoutputproducts'] = []
output_products = querydb.get_processingchain_output_products(process_id)
for outputproduct in output_products:
outputproduct_dict = functions.row2dict(outputproduct)
# outputproduct_dict = outputproduct
# del outputproduct_dict['_labels']
mapset_dict['mapsetoutputproducts'].append(outputproduct_dict)
prod_dict['productmapsets'].append(mapset_dict)
products_dict_all.append(prod_dict)
prod_json = json.dumps(products_dict_all,
ensure_ascii=False,
sort_keys=True,
indent=4,
separators=(', ', ': '))
# logger.info("Processing chains: %s", processingchain_products)
# for row in processingchain_products:
# logger.info("row.dict: %s", row.__dict__)
# logger.info("row.process_id: %s", row.process_id)
# logger.info("row.output_mapsetcode: %s", row.output_mapsetcode)
# logger.info("row.mapsetcode: %s", row.mapsetcode)
# print row.process_id
# print row.output_mapsetcode
# print row.mapsetcode
self.assertEqual(1, 1)
def Test_get_spirits(self):
spirits = querydb.get_spirits()
if spirits.__len__() > 0:
for row in spirits:
row_dict = functions.row2dict(row)
print row_dict
self.assertEqual(1, 1)
def test_row2dict(self):
categories_dict_all = []
categories = querydb.get_categories(allrecs=True)
if hasattr(categories, "__len__") and categories.__len__() > 0:
for row in categories:
row_dict = functions.row2dict(row)
self.assertIsInstance(row_dict, dict)
categories_dict = {
'category_id': row_dict['category_id'],
'descriptive_name': row_dict['descriptive_name']
}
categories_dict_all.append(categories_dict)
self.assertEqual(len(categories_dict_all), 8)
def test_data_management(self):
import json
db_products = querydb.get_products(activated=True)
if db_products.__len__() > 0:
products_dict_all = []
# loop the products list
for row in db_products:
prod_dict = functions.row2dict(row)
productcode = prod_dict['productcode']
version = prod_dict['version']
p = Product(product_code=productcode, version=version)
# print productcode
# does the product have mapsets AND subproducts?
all_prod_mapsets = p.mapsets
all_prod_subproducts = p.subproducts
if all_prod_mapsets.__len__(
) > 0 and all_prod_subproducts.__len__() > 0:
prod_dict['productmapsets'] = []
for mapset in all_prod_mapsets:
mapset_dict = []
# print mapset
mapset_info = querydb.get_mapset(mapsetcode=mapset,
allrecs=False)
# if mapset_info.__len__() > 0:
mapset_dict = functions.row2dict(mapset_info)
# else:
# mapset_dict['mapsetcode'] = mapset
mapset_dict['mapsetdatasets'] = []
all_mapset_datasets = p.get_subproducts(mapset=mapset)
for subproductcode in all_mapset_datasets:
# print 'productcode: ' + productcode
# print 'version: ' + version
# print 'subproductcode: ' + subproductcode
dataset_info = querydb.get_subproduct(
productcode=productcode,
version=version,
subproductcode=subproductcode)
# print dataset_info
# dataset_info = querydb.db.product.get(productcode, version, subproductcode)
# dataset_dict = {}
if dataset_info is not None:
dataset_dict = functions.row2dict(dataset_info)
# dataset_dict = dataset_info.__dict__
# del dataset_dict['_labels']
if hasattr(dataset_info, 'frequency_id'):
if dataset_info.frequency_id == 'e15minute' or dataset_info.frequency_id == 'e30minute':
dataset_dict[
'nodisplay'] = 'no_minutes_display'
# To be implemented in dataset.py
elif dataset_info.frequency_id == 'e1year':
dataset_dict[
'nodisplay'] = 'no_minutes_display'
else:
dataset = p.get_dataset(
mapset=mapset,
sub_product_code=subproductcode)
completeness = dataset.get_dataset_normalized_info(
)
dataset_dict[
'datasetcompleteness'] = completeness
dataset_dict['nodisplay'] = 'false'
dataset_dict['mapsetcode'] = mapset_dict[
'mapsetcode']
dataset_dict[
'mapset_descriptive_name'] = mapset_dict[
'descriptive_name']
mapset_dict['mapsetdatasets'].append(
dataset_dict)
else:
pass
prod_dict['productmapsets'].append(mapset_dict)
products_dict_all.append(prod_dict)
prod_json = json.dumps(products_dict_all,
ensure_ascii=False,
sort_keys=True,
indent=4,
separators=(', ', ': '))
datamanagement_json = '{"success":"true", "total":'\
+ str(db_products.__len__())\
+ ',"products":'+prod_json+'}'
else:
datamanagement_json = '{"success":false, "error":"No data sets defined!"}'
def generateLegendHTML(legend_id):
legendHTMLHorizontal = ''
legendHTMLVertical = ''
legendName = ''
TotSteps = 0
TotColorLabels = 0
TotGroupLabels = 0
legends_HTML = {'legendHTML': '', 'legendHTMLVertical': ''}
if legend_id == None:
return legends_HTML
# print('legend_id: ' + str(legend_id))
legend_info = querydb.get_legend_info(legendid=legend_id)
if hasattr(legend_info, "__len__") and legend_info.__len__() < 1:
return legends_HTML
legend_steps = querydb.get_legend_steps(legendid=legend_id)
if hasattr(legend_steps, "__len__") and legend_steps.__len__() < 1:
return legends_HTML
getTotSteps = querydb.get_legend_totals(legendid=legend_id)
if hasattr(getTotSteps, "__len__") and getTotSteps.__len__() < 1:
return legends_HTML
for row in getTotSteps:
TotSteps = row['totsteps']
TotColorLabels = row['totcolorlabels']
TotGroupLabels = row['totgrouplabels']
for row in legend_info:
# legend_dict = functions.row2dict(row)
legendName = row['legend_name']
######################################
# Create horizontal legend
######################################
if TotSteps >= 35:
fontSizeLabels = 16
fontSizeTitle = 18
# stepWidth = 28
legendWidth = 500 # 775
legendTableBegin = '<table style="border-spacing:0px; background:white; padding: 5px 35px 5px 5px;"> '
legendHeader = '<tr><td colspan=' + str(
TotSteps
) + ' style="font-weight: bold; background:white; padding: 0px 5px 5px 0px; font-size:' + str(
fontSizeTitle) + 'px;">' + legendName + '</td></tr>'
if TotColorLabels > 0:
ColumnSpan = math.ceil(TotSteps / float(TotColorLabels))
else:
ColumnSpan = 1
if TotSteps > 60:
stepWidth = legendWidth / TotSteps
else:
stepWidth = 50
if stepWidth < 4:
stepWidth = 1
legendColors = '<tr style="border-spacing: 0px;">'
for row in legend_steps:
# convert row['color_rgb'] from RGB to html color
color_rgb = row.color_rgb.split(' ')
color_html = functions.rgb2html(color_rgb)
r = color_rgb[0]
g = color_rgb[1]
b = color_rgb[2]
color_html = 'rgb(' + r + ',' + g + ',' + b + ')'
border = ""
if TotSteps <= 30:
border = "border:1px solid black; "
legendColors = legendColors + '<td width=' + str(
stepWidth
) + 'px; height=15px; style="' + border + ' background-color: ' + color_html + '"></td>'
legendColors += '</tr>'
legendColorLabels = '<tr style="border-spacing: 0px;">'
skipcolumns = 0
ColSpan = ColumnSpan
if ColumnSpan > 15:
ColSpan = 10
labelcounter = 0
rowcounter = 0
# ColSpan = 5
for row in legend_steps:
rowcounter += 1
if row.color_label != None and row.color_label.strip() != '':
labelcounter += 1
if labelcounter == TotColorLabels:
ColSpan = TotSteps - rowcounter
legendColorLabels += '<td colspan="' + str(
int(ColSpan)
) + '" style="font-weight: bold; font-size:' + str(
fontSizeLabels) + 'px; line-height:24px; max-width:' + str(
stepWidth +
5) + 'px;" align="left">' + row.color_label + '</td>'
skipcolumns = int(ColSpan - 1)
elif skipcolumns > 0:
skipcolumns -= 1
else:
legendColorLabels += '<td width="1px;"></td>'
legendColorLabels += '</tr>'
legendTableEnd = '</table>'
legendHTMLHorizontal = legendTableBegin + legendHeader + legendColors + legendColorLabels + legendTableEnd
else:
mainTable = '<table style="background: white; border:0px solid black; border-spacing:0px; border-padding:0px; cellspacing:0px; cellpadding:0px; margin: 0px; padding: 10px; ">'
mainTable += '<tr><td colspan=1 style="font-weight: bold; font-size:18px; line-height:24px !important;">' + legendName + '</td></tr>' # line-height:26px;
mainTable += '<tr><td>'
mainTable += '<table class="table table-striped table-header-rotated">'
mainTableBody = '<tbody>'
colorRow = '<tr><td width=25px; height=22px;></td>'
tickRow = '<tr><td width=25px; height=6px;></td>'
footer = '<tfoot><tr>'
row_counter = 0
for row in legend_steps:
row_counter += 1
if row_counter == 1:
borderleft = ' border-left:1px solid black; '
else:
borderleft = ''
borderright = ''
absoluteMaxHeader = ''
if row_counter == legend_steps.__len__():
borderright = ' border-right:1px solid black; '
if row.group_label != None and row.group_label.strip() != '':
absoluteMaxHeader = '<th class="rotate-45"><div><span>' + row.group_label + '</span></div></th>'
# Add color column
# convert row['color_rgb'] from RGB to html color
color_rgb = row.color_rgb.split(' ')
# color_html = functions.rgb2html(color_rgb)
r = color_rgb[0]
g = color_rgb[1]
b = color_rgb[2]
color_html = 'rgb(' + r + ',' + g + ',' + b + ')'
colorRow += '<td width=25px; height=22px; style="border-top:1px solid black; ' + borderleft + borderright + ' background-color: ' + color_html + '"></td>'
tickRow += '<td width=25px; height=6px; style="border-top:1px solid black; border-right:1px solid black; ' + borderleft + ' "></td>'
if row.color_label == None:
color_label = ''
else:
color_label = row.color_label.strip()
footer += '<th class="rotate-45"><div><span>' + color_label + '</span></div></th>' # style="line-height:65px !important;"
footer += absoluteMaxHeader
colorRow += '</tr>'
tickRow += '</tr>'
footer += '</tr></tfoot>'
mainTableBody += colorRow + tickRow + '</tbody>'
mainTable += mainTableBody + footer + '</table></td></tr></table>'
legendHTMLHorizontal = mainTable
######################################
# Create vertical legend
######################################
if TotSteps >= 25:
fontSizeLabels = 16
fontSizeTitle = 18
stepWidth = 28
stepHeight = 1
if TotSteps <= 35:
stepHeight = 15
elif TotSteps <= 65:
stepHeight = 5
elif TotSteps <= 115:
stepHeight = 3
mainTableBegin = '<table style="border-spacing:0px; background:white; padding:0;"> '
mainTableEnd = '</table>'
legendHeaderRow = '<tr><td style="font-weight: bold;background:white; padding: 5px; font-size:' + str(
fontSizeTitle) + 'px;">' + legendName + '</td></tr>'
legendTableBegin = '<table style="border-spacing:0px; background:white; padding: 5px 5px 10px 10px;"> '
legendTableEnd = '</table>'
if TotColorLabels > 0:
ColumnSpan = math.ceil(TotSteps / float(TotColorLabels))
else:
ColumnSpan = 1
Counter = 0
for row in legend_steps:
# Add color column
# convert row['color_rgb'] from RGB to html color
color_rgb = row.color_rgb.split(' ')
color_html = functions.rgb2html(color_rgb)
r = color_rgb[0]
g = color_rgb[1]
b = color_rgb[2]
color_html = 'rgb(' + r + ',' + g + ',' + b + ')'
border = ""
if TotSteps <= 24:
border = "border:1px solid black; "
legendColorColumn = '<td width=' + str(
stepWidth
) + 'px; height=' + str(
stepHeight
) + 'px; style="' + border + ' background-color: ' + color_html + '"></td>'
# Add label column
Counter += 1
legendColorLabelColumn = '<td height="1px;"></td>'
if ColumnSpan > 1:
if row.color_label != None and row.color_label.strip() != '':
# legendColorLabelColumn = '<td rowspan="' + str(ColumnSpan) + '" style="font-size:9px; line-height:10px; " valign="top" align="center">'+row.color_label+'</td>'
legendColorLabelColumn = '<td rowspan=5 style="font-weight: bold; font-size:' + str(
fontSizeLabels
) + 'px; line-height:10px; " valign="top" align="left">' + row.color_label + '</td>'
else:
legendstep_dict = functions.row2dict(row)
if row.color_label != None and row.color_label.strip() != '':
# ColorLabel = ' '+legendstep_dict['color_label'].strip()+' '
legendColorLabelColumn = '<td rowspan="' + str(
ColumnSpan
) + '" style="font-weight: bold; font-size:' + str(
fontSizeLabels
) + 'px; line-height:10px; " align="left">' + row.color_label + '</td>'
legendHTMLVertical += '<tr>' + legendColorColumn + legendColorLabelColumn + '</tr>'
legendHTMLVertical = mainTableBegin + legendHeaderRow + '<tr><td>' + legendTableBegin + legendHTMLVertical + legendTableEnd + '</td></tr>' + mainTableEnd
else:
mainTableBackgroundColor = 'transparent'
fontSizeHeader = 18
fontSizeLabels = 16
firstColumnWidth = 35
legendColorTableBackgroundColor = 'white'
legendLabelTableBackgroundColor = 'white'
extraFirstRowHeight = 14
absoluteMaxRowColorTableHeight = 18
colorColumnWidth = 35
colorColumnHeight = 20
tickColumnWidth = 8
tickColumnHeight = 20
labelColumnHeight = 20
mainTableBegin = '<table style="background: ' + mainTableBackgroundColor + '; border:0px solid black; border-spacing:0px; border-padding:0px; cellspacing=0px; cellpadding=0px; margin: 0px; padding: 0px; ">'
legendHeaderRow = '<tr><td colspan=2 style="font-weight: bold; background: white; padding:3px;"><span style=" font-size:' + str(
fontSizeHeader
) + 'px;">' + legendName + '</span></td></tr>' # line-height: 24px;
legendRowBegin = '<tr>'
firstColumnBegin = '<td width=' + str(
firstColumnWidth
) + 'px; style="border:0px solid black; border-spacing:0px; border-padding:0px; cellspacing=0px; cellpadding=0px; margin: 0px; padding: 0px;">'
secondColumnBegin = '<td valign="top" align="left" style="border:0px solid black; border-spacing:0px; border-padding:0px; cellspacing=0px; cellpadding=0px; margin: 0px; padding: 0px;" >'
legendColorTable = '<table style="background: ' + legendColorTableBackgroundColor + '; border:0px solid black; border-spacing:0px; border-padding:0px; cellspacing=0px; cellpadding=0px; margin: 0px; padding-left: 3px;">'
legendColorTable += '<tr><td colspan=2 height=' + str(
extraFirstRowHeight) + 'px; style=""></td></tr>'
legendLabelTable = '<table style="background: ' + legendLabelTableBackgroundColor + '; border:0px solid black; border-spacing:0px; border-padding:0px; cellspacing=0px; cellpadding=0px; margin: 0px; padding: 3px; ">'
row_counter = 0
# print('legend_steps: ' + str(legend_steps.__len__()))
for row in legend_steps:
row_counter += 1
if row_counter == 1:
bordertop = ' border-top:1px solid black; '
else:
bordertop = ' '
borderbottom = ' '
borderbottomtick = ' '
absoluteMaxRowColorTable = ''
absoluteMaxRowLegendLabelTable = ''
if row_counter == legend_steps.__len__():
borderbottom = ' border-bottom:1px solid black; '
if row.group_label != None and row.group_label.strip() != '':
borderbottomtick = ' border-bottom:1px solid black; '
absoluteMaxRowColorTable = '<tr><td colspan=2 height=' + str(
absoluteMaxRowColorTableHeight
) + 'px; style=""></td></tr>'
absoluteMaxRowLegendLabelTable = '<tr><td height=' + str(
labelColumnHeight
) + 'px; style="font-weight: bold; font-size:' + str(
fontSizeLabels
) + 'px; padding:0px; margin: 0px; padding-left:5px;" valign="middle" align="left">' + row.group_label + '</td></tr>'
# Add color column
# convert row['color_rgb'] from RGB to html color
color_rgb = row.color_rgb.split(' ')
# color_html = functions.rgb2html(color_rgb)
r = color_rgb[0]
g = color_rgb[1]
b = color_rgb[2]
color_html = 'rgb(' + r + ',' + g + ',' + b + ')'
legendColorTable += '<tr> ' + \
'<td width='+str(colorColumnWidth)+'px; height='+str(colorColumnHeight)+'px; style="padding:0px; margin: 0px; border-left:1px solid black; '+borderbottom+bordertop+' background-color: '+color_html+'"></td>' + \
'<td width='+str(tickColumnWidth)+'px; height='+str(tickColumnHeight)+'px; style="padding:0px; margin: 0px; border-top:1pt solid black; border-left:1px solid black; '+borderbottomtick+'"></td>' + \
'</tr>'
legendColorTable += absoluteMaxRowColorTable
if row.color_label == None:
color_label = ''
else:
color_label = row.color_label.strip()
legendLabelTable += '<tr><td height=' + str(
labelColumnHeight
) + 'px; style="font-weight: bold; font-size:' + str(
fontSizeLabels
) + 'px; padding:0px; margin: 0px; padding-left:5px;" valign="middle" align="left">' + color_label + '</td></tr>'
legendLabelTable += absoluteMaxRowLegendLabelTable
legendColorTable += '</table>'
# legendLabelTable += '<tr><td height=' + str(extraFirstRowHeight) + 'px; style=""></td></tr>'
legendLabelTable += '</table>'
columnEnd = '</td>'
legendRowEnd = '</tr>'
mainTableEnd = '</table>'
legendHTMLVertical = mainTableBegin + legendHeaderRow + legendRowBegin + firstColumnBegin + legendColorTable + columnEnd + secondColumnBegin + legendLabelTable + columnEnd + legendRowEnd + mainTableEnd
# print legendHTMLVertical
legends_HTML = {
'legendHTML': legendHTMLHorizontal,
'legendHTMLVertical': legendHTMLVertical
}
return legends_HTML
def Test_get_internetsources(self):
internetsources = querydb.get_internetsources()
if internetsources.__len__() > 0:
for row in internetsources:
row_dict = functions.row2dict(row)
def Test_get_eumetcastsources(self):
eumetcastsources = querydb.get_eumetcastsources()
if eumetcastsources.__len__() > 0:
for row in eumetcastsources:
row_dict = functions.row2dict(row)
def push_data_ftp(dry_run=False,
user=None,
psw=None,
url=None,
trg_dir=None,
masked=True):
# Synchronized data towards an ftp server (only for JRC)
# It replaces, since the new srv-ies-ftp.jrc.it ftp is set, the bash script: mirror_to_ftp.sh
# Configuration: it looks at all 'non-masked' products and pushes them
# For the mapsets, find what is in the filesystem, and pushes only the 'largest'
# It uses a command like:
# lftp -e "mirror -RLe /data/processing/vgt-ndvi/sv2-pv2.1/SPOTV-Africa-1km/derived/10dmax-linearx2/
# /narma/eStation_2.0/processing/vgt-ndvi/sv2-pv2.1/SPOTV-Africa-1km/derived/10dmax-linearx2/;exit"
# -u narma:JRCVRw2960 sftp://srv-ies-ftp.jrc.it"" >> /eStation2/log/push_data_ftp.log
#
spec_logger = log.my_logger('apps.es2system.push_data_ftp')
try:
from config import server_ftp
except:
logger.warning('Configuration file for ftp sync not found. Exit')
return 1
if user is None:
user = server_ftp.server['user']
if psw is None:
psw = server_ftp.server['psw']
if url is None:
url = server_ftp.server['url']
if trg_dir is None:
trg_dir = server_ftp.server['data_dir']
# Create an ad-hoc file for the lftp command output (beside the standard logger)
logfile = es_constants.es2globals['log_dir'] + 'push_data_ftp.log'
message = time.strftime(
"%Y-%m-%d %H:%M") + ' INFO: Running the ftp sync now ... \n'
logger.debug("Entering routine %s" % 'push_data_ftp')
# Loop over 'not-masked' products
products = querydb.get_products(masked=False)
# produts=products[21:23] # test a subset
for row in products:
prod_dict = functions.row2dict(row)
productcode = prod_dict['productcode']
version = prod_dict['version']
spec_logger.info('Working on product {}/{}'.format(
productcode, version))
# TEMP - For testing only
# if productcode!='vgt-ndvi' or version !='sv2-pv2.2':
# continue
# Check it if is in the list of 'exclusions' defined in ./config/server_ftp.py
key = '{}/{}'.format(productcode, version)
skip = False
if key in server_ftp.exclusions:
skip = True
logger.debug('Do not sync for {}/{}'.format(productcode, version))
p = Product(product_code=productcode, version=version)
all_prod_mapsets = p.mapsets
all_prod_subproducts = p.subproducts
# Check there is at least one mapset and one subproduct
if all_prod_mapsets.__len__() > 0 and all_prod_subproducts.__len__(
) > 0 and not skip:
# In case of several mapsets, check if there is a 'larger' one
if len(all_prod_mapsets) > 1:
mapset_to_use = []
for my_mapset in all_prod_mapsets:
mapset_info = querydb.get_mapset(mapsetcode=my_mapset,
allrecs=False)
if hasattr(mapset_info, "mapsetcode"):
my_mapobj = MapSet()
my_mapobj.assigndb(my_mapset)
larger_mapset = my_mapobj.get_larger_mapset()
if larger_mapset is not None:
if larger_mapset not in mapset_to_use:
mapset_to_use.append(larger_mapset)
else:
if my_mapset not in mapset_to_use:
mapset_to_use.append(my_mapset)
else:
mapset_to_use = all_prod_mapsets
# Loop over existing mapset
for mapset in mapset_to_use:
all_mapset_datasets = p.get_subproducts(mapset=mapset)
# Loop over existing subproducts
for subproductcode in all_mapset_datasets:
# Get info - and ONLY for NOT masked products
dataset_info = querydb.get_subproduct(
productcode=productcode,
version=version,
subproductcode=subproductcode,
masked=masked) # -> TRUE means only NOT masked sprods
if dataset_info is not None:
dataset_dict = functions.row2dict(dataset_info)
dataset_dict['mapsetcode'] = mapset
logger.debug('Working on {}/{}/{}/{}'.format(
productcode, version, mapset, subproductcode))
subdir = functions.set_path_sub_directory(
productcode, subproductcode,
dataset_dict['product_type'], version, mapset)
source = data_dir + subdir
target = trg_dir + subdir
# command = 'lftp -e "mirror -RLe {} {};exit" -u {}:{} {}"" >> {}'.format(source,target,user,psw,url,logfile)
command = 'lftp -e "mirror -RLe {} {};exit" -u {}:{} {}"" >> /dev/null'.format(
source, target, user, psw, url)
logger.debug("Executing %s" % command)
spec_logger.info(
'Working on mapset/subproduct {}/{} \n'.format(
mapset, subproductcode))
# return
try:
status = os.system(command)
if status:
logger.error("Error in executing %s" % command)
spec_logger.error("Error in executing %s" %
command)
except:
logger.error(
'Error in executing command: {}'.format(
command))
spec_logger.error(
'Error in executing command: {}'.format(
command))
def Test_get_internetsources(self):
internetsources = querydb.get_internetsources()
if internetsources.__len__() > 0:
for row in internetsources:
row_dict = functions.row2dict(row)
def Test_get_eumetcastsources(self):
eumetcastsources = querydb.get_eumetcastsources()
if eumetcastsources.__len__() > 0:
for row in eumetcastsources:
row_dict = functions.row2dict(row)