def check_all():
dcm = pytroll_db.DCManager(DB)
file_list = dcm.get_files()
load_logins()
for filename in file_list:
uris_remove = []
for uri in filename.uris:
try:
is_there = cached_check_file_ssh(uri.uri)
except ConnectionError:
print "Can't connect, skipping", uri.uri
continue
if not is_there:
print "Missing:", uri.uri
uris_remove.append(uri)
else:
print "Present:", uri.uri
if len(uris_remove) == len(filename.uris):
print "removing", filename
dcm.delete(filename)
for uri in uris_remove:
print "removing", uri
dcm.delete(uri)
dcm.save()
def threaded_check_all(remove_unaccessible=False):
dcm = pytroll_db.DCManager(DB)
file_list = dcm.get_files()
load_logins()
print "requesting..."
result = is_there_dict(file_list)
cnt = 0
removed = 0
print "checking..."
for filename in file_list:
uris_remove = []
for uri in filename.uris:
try:
is_there = result[uri.uri]
cnt += 1
if is_there is None:
if remove_unaccessible:
removed += 1
uris_remove.append(uri)
elif not is_there:
removed += 1
uris_remove.append(uri)
except KeyError:
print "error, skipping", uri.uri
if len(uris_remove) == len(filename.uris):
dcm.delete(filename)
for uri in uris_remove:
dcm.delete(uri)
print "removing", uri
dcm.save()
print "Removed", removed, "out of", cnt
def clean_old_entries_in_db(file_type_name, timestamp_utc):
"""Clean the DB for products of a given type with creation time older
than a given threshold"""
dcm = pytroll_db.DCManager(DB)
file_list = dcm.get_files(file_type_name,
newest_creation_time=timestamp_utc)
for file_obj in file_list:
dcm.delete(file_obj)
dcm.save()
def sync_product_db(file_type_name, dirname, glob_filter):
# Syncronize DB content with filesystem so no files exist in DB which are
# not present in filesystem
filename_list = [
os.path.basename(item)
for item in glob.glob('%s/%s' % (dirname, glob_filter))
]
print 'filenames: ', filename_list
dcm = pytroll_db.DCManager(DB)
file_type = dcm.get_file_type(file_type_name)
file_obj_list = file_type.file_objs
for file_obj in file_obj_list:
if file_obj.filename not in filename_list:
dcm.delete(file_obj)
dcm.save()
#!/usr/bin/env python
#
#
#
#
"""Insert test values into database through SQLAlchemy interface """
from datetime import datetime, timedelta
import shapely
import pytroll_db as db
dcm = db.DCManager('postgresql://a001673:@localhost.localdomain:5432/sat_db')
dcm.create_parameter_type(1, 'dummy', 'parameter_value')
hrpt = dcm.create_file_type(1, 'HRPT', 'HRPT level 0 data')
hirlam = dcm.create_file_type(2, 'smhi_hirlam', 'HIRLAM data')
hrit = dcm.create_file_type(3, "HRIT", "met 9 hrit data")
hrpt1b = dcm.create_file_type(4, 'HRPT 1b', 'HRPT level 1b data')
hrpt_L0 = dcm.create_file_format(1, 'hrpt level 0', '')
grib = dcm.create_file_format(2, 'GRIB', '')
eps_1b = dcm.create_file_format(3, 'EPS 1b', 'EPS format, level 1b')
p_tofs = dcm.create_parameter(1, 1, 'time_of_first_scanline', '')
p_tols = dcm.create_parameter(2, 1, 'time_of_last_scanline', '')
p_on = dcm.create_parameter(3, 1, 'orbit_number', '')
p_satname = dcm.create_parameter(4, 1, 'satellite_name', '')
p_toa = dcm.create_parameter(5, 1, 'time_of_analysis', '')
p_fl = dcm.create_parameter(6, 1, 'forcast_length', '')
p_pc = dcm.create_parameter(7, 1, 'processing_center', '')
p_sst = dcm.create_parameter(8, 1, 'sub_satellite_track', '')
def delete_product(filename):
dcm = pytroll_db.DCManager(DB)
file_obj = dcm.get_file(filename)
dcm.delete(file_obj)
dcm.save()
if i == 0:
value += '%s %s' % (item[0], item[1])
else:
value += ' ,%s %s' % (item[0], item[1])
value += ')'
wkt_o = shapely.wkt.loads(value)
p_track = dcm.get_parameter('sub_satellite_track')
pls = dcm.create_parameter_linestring(wkt_o, file_obj=nf, parameter=p_track)
dcm.save()
if __name__ == '__main__':
if len(sys.argv) < 2:
print "Usage: %s <path> <glob-string>" % sys.argv[0]
sys.exit(0)
dcm = pytroll_db.DCManager('postgresql://*****:*****@safe:5432/sat_db')
dirname, file_filter = sys.argv[1:3]
tmp = '%s/%s' % (dirname, file_filter)
print tmp
files = glob.glob('%s/%s' % (dirname, file_filter))
#print 'files: ', files
for file_path in files:
try:
add_product(dcm, os.path.basename(file_path))
except Exception, e:
print 'Failed to add file %s to database: %s' % (os.path.basename(file_path), e)
dcm.rollback()
break
import pytroll_db
dcm = pytroll_db.DCManager(
'postgresql://*****:*****@devsat-lucid:5432/testdb2')
#dcm = pytroll_db.DCManager('postgresql://a000680:@localhost.localdomain:5432/sat_db')
ft = dcm.get_file_type('HRPT')
ff = dcm.get_file_format('hrpt level 0')
nf = dcm.create_file('test100', file_type=ft, file_format=ff)
nf1 = dcm.create_file('test101',
file_type_name='HRPT',
file_format_name='hrpt level 0')
print nf
p = dcm.get_parameter('orbit_number')
pv = dcm.create_parameter_value(666, file_obj=nf, parameter=p)
pv = dcm.create_parameter_value(666,
uid='test101',
parameter_name='orbit_number')
value = 'LINESTRING (3 3, 4 4, 5 5, 6 6)'
import shapely
wkt_o = shapely.wkt.loads(value)
p_track = dcm.get_parameter('sub_satellite_track')
pls = dcm.create_parameter_linestring(wkt_o, file_obj=nf, parameter=p_track)
pls = dcm.create_parameter_linestring(wkt_o,
uid="test100",
parameter_name="sub_satellite_track")