def run(self):
utils.valid_path(self.output_array_path)
rasters = utils.rasters_stack(self.dates, self.muni_raster_path,
'.tiff')
pickle_path = self.output_array_path + self.cve_muni + '_' + self.dates[
0] + '-' + self.dates[-1] + '.p'
pickle.dump(rasters, open(pickle_path, 'wb'))
def requires(self):
utils.valid_path(self.target_raster_path)
for date in self.dates:
yield CropDate(cve_muni=self.cve_muni,
date=date,
target_raster_path=self.target_raster_path,
output_raster_path=self.muni_raster_path)
def requires(self):
utils.valid_path(self.muni_rasters)
dates = utils.get_valid_dates(self.start, self.end)
munis = pu.get_munis(self.edos)
for cve_muni in munis:
muni_raster_path = self.muni_rasters + cve_muni + '/'
utils.valid_path(muni_raster_path)
yield MunArray(cve_muni=cve_muni,
dates=dates,
muni_raster_path=muni_raster_path)
def remove_changed (databank, lines=[]):
pattern = parse_regex(databank['regex'])
line_matches = {}
if databank ['crawltype'] == LINE:
for line in lines:
m = pattern.search (line)
if m:
line_matches [m.group (1)] = line
# Remove entries where the file's mtime has changed or where the
# actual file/line was removed or doesn't match the pattern anymore:
for entry in get_present_entries (databank['name']):
path = entry ['filepath']
if databank ['crawltype'] == FILE and \
(not valid_path (databank['name'], path) or \
os.path.getmtime (path) != entry['mtime']):
storage.remove ('entries', {'databank_name': databank['name'], 'pdbid': entry['pdbid']})
elif databank ['crawltype'] == LINE and \
(not os.path.isfile (path) or \
os.path.getmtime (path) != entry['mtime'] or \
entry ['pdbid'] not in line_matches):
storage.remove ('entries', {'databank_name': databank['name'], 'pdbid': entry['pdbid']})
def remove_changed(databank, lines=[]):
pattern = parse_regex(databank['regex'])
line_matches = {}
if databank['crawltype'] == LINE:
for line in lines:
m = pattern.search(line)
if m:
line_matches[m.group(1)] = line
# Remove entries where the file's mtime has changed or where the
# actual file/line was removed or doesn't match the pattern anymore:
for entry in get_present_entries(databank['name']):
path = entry['filepath']
if databank ['crawltype'] == FILE and \
(not valid_path (databank['name'], path) or \
os.path.getmtime (path) != entry['mtime']):
storage.remove('entries', {
'databank_name': databank['name'],
'pdbid': entry['pdbid']
})
elif databank ['crawltype'] == LINE and \
(not os.path.isfile (path) or \
os.path.getmtime (path) != entry['mtime'] or \
entry ['pdbid'] not in line_matches):
storage.remove('entries', {
'databank_name': databank['name'],
'pdbid': entry['pdbid']
})
if 'filepath' not in entries_psql [ID]:
entries_psql [ID]['comment'] = text
comments_psql [ID] = text
for ID in files_psql:
databank_name, pdbid = ID.split (',')
path = files_psql [ID]
if ID in files_mongo:
if files_mongo [ID] != files_psql [ID] and \
not valid_path (databank_name, files_mongo [ID]):
print "different file path in mongo", ID, ':', files_mongo [ID], 'vs.', files_psql [ID]
else:
if has_present_parent (databank_name, pdbid) and \
valid_path (databank_name, path):
print 'file absent in mongo:', ID, path
for ID in comments_psql:
databank_name, pdbid = ID.split (',')
comment = comments_psql [ID]
if ID in comments_mongo:
ID = '%s,%s' % (databank_name, pdbid)
if 'filepath' not in entries_psql[ID]:
entries_psql[ID]['comment'] = text
comments_psql[ID] = text
for ID in files_psql:
databank_name, pdbid = ID.split(',')
path = files_psql[ID]
if ID in files_mongo:
if files_mongo [ID] != files_psql [ID] and \
not valid_path (databank_name, files_mongo [ID]):
print "different file path in mongo", ID, ':', files_mongo[
ID], 'vs.', files_psql[ID]
else:
if has_present_parent (databank_name, pdbid) and \
valid_path (databank_name, path):
print 'file absent in mongo:', ID, path
for ID in comments_psql:
databank_name, pdbid = ID.split(',')
comment = comments_psql[ID]