class AnswerHttpPing_withCache:
def __init__(self, path_db=None):
self.httping = HttpInfo()
table_name = "httping"
if path_db:
self.cache = CacheManager(name_table=table_name,
path_database=path_db)
else:
self.cache = CacheManager(name_table=table_name,
path_database=os.path.join(
os.getcwd(), "db.sqlite"))
self.cache.setup("CREATE TABLE %s ("
"_url TEXT PRIMARY KEY, "
"code TEXT, "
"size TEXT, "
"duration TEXT);" % table_name)
def query(self, url):
from_cache = self.cache.retrieve_from_cache("_url", url)
if from_cache:
Logger.debug("from cache for %s" % url)
return from_cache
else:
from_query = self.httping.query(url)
Logger.debug("from query for %s" % url)
if from_query:
terms = [
from_query['url'],
str(from_query['code']),
str(from_query['size']),
str(from_query['duration'])
]
self.cache.insert_to_cache(terms)
return terms
def __init__(self, source=None, logger=None):
# General Attributes
self.source = source
self.reader = None
self.drp = None
self.count = 0
self.Mcount = 99999
if source is not None:
self.ingestDir = PXPaths.RXQ + self.source.name
if self.source.type == 'filter' or self.source.type == 'filter-bulletin' :
self.ingestDir = PXPaths.FXQ + self.source.name
self.logger = source.logger
elif logger is not None:
self.logger = logger
self.pxManager = PXManager() # Create a manager
self.pxManager.setLogger(self.logger) # Give the logger to the the manager
self.pxManager.initNames() # Set rx and tx names
self.clientNames = self.pxManager.getTxNames() # Obtains the list of client's names (the ones to wich we can link files)
self.filterNames = self.pxManager.getFxNames() # Obtains the list of filter's names (the ones to wich we can link files)
if source is not None:
if self.source.name in self.filterNames : self.filterNames.remove(self.source.name)
self.sourlientNames = self.pxManager.getTRxNames() # Obtains the list of sourlient's names (the ones to wich we can link files)
self.allNames = self.clientNames + self.filterNames + self.sourlientNames # Clients + Sourlients names
self.clients = {} # All the Client/Filter/Sourlient objects
self.fileCache = None # product processed.
self.dbDirsCache = CacheManager(maxEntries=200000, timeout=25*3600) # Directories created in the DB
self.clientDirsCache = CacheManager(maxEntries=25000, timeout=2*3600) # File ingested in RXQ
self.feedNames = [] # source to feed
self.feeds = {}
if source is not None:
self.logger.info("Ingestor (source %s) can link files to clients: %s" % (source.name, self.allNames))
def __init__(self, client, logger):
self.client = client # Client object (give access to all configuration options)
self.timeout = client.timeout # No timeout for now
self.logger = logger # Logger object
self.connection = None # The connection
self.igniter = None
self.ssl = False
self.reader = DiskReader(PXPaths.TXQ + self.client.name,
self.client.batch, self.client.validation,
self.client.patternMatching,
self.client.mtime, True, self.logger,
eval(self.client.sorter), self.client)
self.debugFile = False
self.cacheManager = CacheManager(maxEntries=self.client.cache_size,
timeout=8 * 3600)
# AMQP is there a max for message size
# self.set_maxLength(self.client.maxLength)
# statistics.
self.totBytes = 0
self.initialTime = time.time()
self.finalTime = None
self._connect()
def __init__(self, source=None, logger=None):
# General Attributes
self.source = source
self.reader = None
self.drp = None
self.count = 0
self.Mcount = 99999
if source is not None:
self.ingestDir = PXPaths.RXQ + self.source.name
if self.source.type == 'filter' or self.source.type == 'filter-bulletin' :
self.ingestDir = PXPaths.FXQ + self.source.name
self.logger = source.logger
elif logger is not None:
self.logger = logger
self.pxManager = PXManager() # Create a manager
self.pxManager.setLogger(self.logger) # Give the logger to the the manager
self.pxManager.initNames() # Set rx and tx names
self.clientNames = self.pxManager.getTxNames() # Obtains the list of client's names (the ones to wich we can link files)
self.filterNames = self.pxManager.getFxNames() # Obtains the list of filter's names (the ones to wich we can link files)
if source is not None:
if self.source.name in self.filterNames : self.filterNames.remove(self.source.name)
self.sourlientNames = self.pxManager.getTRxNames() # Obtains the list of sourlient's names (the ones to wich we can link files)
self.allNames = self.clientNames + self.filterNames + self.sourlientNames # Clients + Sourlients names
self.clients = {} # All the Client/Filter/Sourlient objects
self.fileCache = None # product processed.
self.dbDirsCache = CacheManager(maxEntries=200000, timeout=25*3600) # Directories created in the DB
self.clientDirsCache = CacheManager(maxEntries=25000, timeout=2*3600) # File ingested in RXQ
self.feedNames = [] # source to feed
self.feeds = {}
if source is not None:
self.logger.info("Ingestor (source %s) can link files to clients: %s" % (source.name, self.allNames))
class AnswerIPInfo_withCache:
def __init__(self, path_db=None):
self.ipinfo = IPInfo()
if path_db:
self.cache = CacheManager(name_table="ipinfo",
path_database=path_db)
else:
self.cache = CacheManager(name_table="ipinfo",
path_database=os.path.join(
os.getcwd(), "db.sqlite"))
self.cache.setup("CREATE TABLE %s ("
"_hostname TEXT PRIMARY KEY, "
"city TEXT, "
"country TEXT, "
"countryCode TEXT, "
"org TEXT, "
"query TEXT);" % "ipinfo")
def query(self, hostname):
from_cache = self.cache.retrieve_from_cache("_hostname", hostname)
if from_cache:
Logger.debug("from cache for %s" % hostname)
return from_cache
else:
from_query = self.ipinfo.query(hostname)
Logger.debug("from query for %s" % hostname)
if from_query:
terms = [
from_query['hostname'], from_query['city'],
from_query['country'], from_query['countryCode'],
from_query['org'], from_query['query']
]
self.cache.insert_to_cache(terms)
return terms
def update_cache():
"""
update CodeIgniter cache
"""
try:
c = CacheManager()
except:
logging.warning("Skipping CodeIgniter cache update")
return
try:
c.update_pdb_cache()
except:
logging.warning(traceback.format_exc(sys.exc_info()))
logging.warning('Pdb cache update failed')
try:
c.update_nrlist_cache()
except:
logging.warning(traceback.format_exc(sys.exc_info()))
logging.warning('Nrlist cache update failed')
try:
c.update_motif_cache()
except:
logging.warning(traceback.format_exc(sys.exc_info()))
logging.warning('Motif cache update failed')
try:
c.update_loop_cache()
except:
logging.warning(traceback.format_exc(sys.exc_info()))
logging.warning('Loop cache update failed')
async def main():
loop = asyncio.get_event_loop()
a = CacheManager(loop)
a.tset("b", 5, 6)
await asyncio.sleep(3)
print(a.get("b"))
def setUp(self):
self.manager = CacheManager(maxEntries=3, timeout=5 * 3600)
self.files = [
'db/test-file-grib', 'db/test-file-burf', 'db/test-file-bulletin'
]
#f = open(self.files[0])
#self.data = f.read(1048576)
#while len(self.data) :
#print self.data
#self.data = f.read(1048576)
#f.close()
self.data = 'ceci est un test'
def __init__(self, paths_manager, n_partial_ordering):
"""
Constructor
:param paths_manager: manager for paths
:param n_partial_ordering: partial ordering between nodes involved in paths
"""
self.paths_manager = paths_manager
self.n_partial_ordering = n_partial_ordering
self.predicates_to_most_specific_index = CacheManager()
self.most_specific_indices = dict()
def get(url, cache=True, use_cookie=False):
if cache:
data = CacheManager.get(url)
if not data:
data = Server.download(url, use_cookie=use_cookie)
if data:
CacheManager.set(url, data)
return data, False
else:
return data, True
else:
data = Server.download(url, use_cookie=use_cookie)
return data, False
def __init__(self, path_db=None):
self.httping = HttpInfo()
table_name = "httping"
if path_db:
self.cache = CacheManager(name_table=table_name,
path_database=path_db)
else:
self.cache = CacheManager(name_table=table_name,
path_database=os.path.join(
os.getcwd(), "db.sqlite"))
self.cache.setup("CREATE TABLE %s ("
"_url TEXT PRIMARY KEY, "
"code TEXT, "
"size TEXT, "
"duration TEXT);" % table_name)
def __init__(self, path_db=None):
self.ipinfo = IPInfo()
if path_db:
self.cache = CacheManager(name_table="ipinfo",
path_database=path_db)
else:
self.cache = CacheManager(name_table="ipinfo",
path_database=os.path.join(
os.getcwd(), "db.sqlite"))
self.cache.setup("CREATE TABLE %s ("
"_hostname TEXT PRIMARY KEY, "
"city TEXT, "
"country TEXT, "
"countryCode TEXT, "
"org TEXT, "
"query TEXT);" % "ipinfo")
def __init__(self, client, logger):
self.client = client # Client object (give access to all configuration options)
self.remoteHost = client.host # Remote host (name or ip)
self.port = int(
client.port) # Port (int) to which the receiver is bind
self.address = (self.remoteHost, self.port) # Socket address
self.timeout = client.timeout # No timeout for now
self.logger = logger # Logger object
self.socketAMIS = None # The socket
self.igniter = None
self.reader = DiskReader(PXPaths.TXQ + self.client.name,
self.client.batch, self.client.validation,
self.client.patternMatching,
self.client.mtime, True, self.logger,
eval(self.client.sorter), self.client)
self.preamble = chr(curses.ascii.SOH) + "\r\n"
self.endOfLineSep = "\r\r\n"
self.endOfMessage = self.endOfLineSep + chr(
curses.ascii.ETX) + "\r\n\n" + chr(curses.ascii.EOT)
self.debugFile = False
self.cacheManager = CacheManager(maxEntries=self.client.cache_size,
timeout=8 * 3600)
# AMIS's maximum bulletin size is 14000
self.set_maxLength(self.client.maxLength)
# statistics.
self.totBytes = 0
self.initialTime = time.time()
self.finalTime = None
self._connect()
def __init__(self):
QWidget.__init__(self)
ssl._create_default_https_context = ssl._create_unverified_context
self.threadaaa = MyThread(1, 1)
self.threadaaa.start()
self.threadaaa.singal.connect(self.setItemViews)
self.downloadThreads = []
self.setWindowTitle('视频下载')
self.cacheManager = CacheManager()
self.allDataArr = []
# self.requet = Reques()
# self.grid = QGridLayout()
self.listView = QListWidget()
self.container()
def load_cache_managers(cache_managers_dir):
"""
Load cache managers from the given directory
:param cache_managers_dir: directory where cache managers files are stored
:return: a dict with 2 keys representing the 2 cache managers (rdf_nodes and predicates)
"""
if cache_managers_dir[-1] != "/":
cache_managers_dir += "/"
cache_managers = {
"rdf_nodes": CacheManager(),
"predicates": CacheManager()
}
cache_managers["predicates"].load_from_csv(cache_managers_dir + "predicates_cache_manager.csv")
cache_managers["rdf_nodes"].load_from_csv(cache_managers_dir + "rdf_nodes_cache_manager.csv")
return cache_managers
def get_payload(self, gene_symbol, request_type):
# Attempt to pull payload from cache
cs = CacheManager(self.settings['CACHE_FILE'])
cache_key = f'{gene_symbol}_{request_type}'
json_response = cs.fetch(cache_key)
# Otherwise hit the API
if not json_response:
rs = ReqService(self.settings)
json_response = rs.make_request(self.symbol_to_id(gene_symbol),
request_type=request_type).json()
# If request was successful, update cache
if json_response:
cs.update(cache_key, json_response)
else:
sys.exit()
return json_response
def setUp(self):
self.manager = CacheManager(maxEntries=3, timeout=5 * 3600)
self.files = ['db/test-file-grib', 'db/test-file-burf', 'db/test-file-bulletin']
#f = open(self.files[0])
#self.data = f.read(1048576)
#while len(self.data) :
#print self.data
#self.data = f.read(1048576)
#f.close()
self.data = 'ceci est un test'
def setUp(self,logFile='log/SenderFTP.log'):
self.logger = Logger(logFile, 'DEBUG', 'Sub')
self.logger = self.logger.getLogger()
self.client = Client('client-test',self.logger)
self.cacheManager = CacheManager(maxEntries=3, timeout=5 * 3600)
#Le sender doit etre capable de connecter au serveur ftp pour continuer le test.
self.sender = SenderFTP(self.client, self.logger, self.cacheManager)
file1 = open("txq/client-test/3/test","w")
file1.close()
self.files = ['txq/client-test/3/test']
def __init__(self,path,client,logger):
gateway.gateway.__init__(self, path, client, logger)
self.client = client
self.establishConnection()
self.reader = DiskReader(PXPaths.TXQ + self.client.name,
self.client.batch, # Number of files we read each time
self.client.validation, # name validation
self.client.patternMatching, # pattern matching
self.client.mtime, # we don't check modification time
True, # priority tree
self.logger,
eval(self.client.sorter),
self.client)
# Mechanism to eliminate multiple copies of a bulletin
self.cacheManager = CacheManager(maxEntries=self.client.cache_size, timeout=8*3600)
# WMO's maximum bulletin size is 500 000 bytes
self.set_maxLength( self.client.maxLength )
def _startFactory(self):
log.msg('Starting the main apt_p2p application')
self.cache_dir = FilePath(config.get('DEFAULT', 'CACHE_DIR'))
if not self.cache_dir.child(download_dir).exists():
self.cache_dir.child(download_dir).makedirs()
if not self.cache_dir.child(peer_dir).exists():
self.cache_dir.child(peer_dir).makedirs()
self.db = DB(self.cache_dir.child('apt-p2p.db'))
self.dht = DHT(self.dhtClass, self.db)
df = self.dht.start()
df.addCallback(self._dhtStarted)
self.stats = StatsLogger(self.db)
self.http_server = TopLevel(self.cache_dir.child(download_dir),
self.db, self)
self.http_server.getHTTPFactory().startFactory()
self.peers = PeerManager(self.cache_dir.child(peer_dir), self.dht,
self.stats)
self.mirrors = MirrorManager(self.cache_dir)
self.cache = CacheManager(self.cache_dir.child(download_dir), self.db,
self)
def process_file(filename):
parser = InputParser()
parser.parse("in/" + filename + ".in")
manager = CacheManager()
manager.videos = parser.getVideos()
manager.caches = parser.getCaches()
manager.endpoints = parser.getEndPoints()
manager.requests = parser.getRequests()
manager.FillCaches()
outputstring = manager.OutputString()
outputfile = open("out/" + filename + ".out", 'w')
outputfile.write(outputstring)
def __init__(self, ingestor, bullManager, reader):
# General Attributes
self.bullManager = bullManager
self.ingestor = ingestor
self.logger = ingestor.logger
self.reader = reader
self.source = ingestor.source
self.files = []
self.data = []
self.entry = None
self.now = time.mktime(time.localtime())
# a cacheManager to make sure we process a file only once
self.cacheManager = CacheManager(maxEntries=self.source.cache_size,
timeout=int(self.source.history) *
3600)
# reading the collection station config file...
sp = StationParser.StationParser(PXPaths.STATION_TABLE)
sp.parse()
self.mapCollectionStation = sp.getStationsColl()
# instantiate the collection state class
self.collectionState = CollectionState.CollectionState(self)
self.getState = self.collectionState.getState
self.setState = self.collectionState.setState
self.updateState = self.collectionState.updateCollectionState
# instantiate the collection builder class
self.collectionBuilder = CollectionBuilder.CollectionBuilder(self)
self.collectionBuild = self.collectionBuilder.process
def _startFactory(self):
log.msg('Starting the main apt_p2p application')
self.cache_dir = FilePath(config.get('DEFAULT', 'CACHE_DIR'))
if not self.cache_dir.child(download_dir).exists():
self.cache_dir.child(download_dir).makedirs()
if not self.cache_dir.child(peer_dir).exists():
self.cache_dir.child(peer_dir).makedirs()
self.db = DB(self.cache_dir.child('apt-p2p.db'))
self.dht = DHT(self.dhtClass, self.db)
df = self.dht.start()
df.addCallback(self._dhtStarted)
self.stats = StatsLogger(self.db)
self.http_server = TopLevel(self.cache_dir.child(download_dir), self.db, self)
self.http_server.getHTTPFactory().startFactory()
self.peers = PeerManager(self.cache_dir.child(peer_dir), self.dht, self.stats)
self.mirrors = MirrorManager(self.cache_dir)
self.cache = CacheManager(self.cache_dir.child(download_dir), self.db, self)
def __init__(self,
path,
batch=20000,
validation=False,
patternMatching=False,
mtime=0,
prioTree=False,
logger=None,
sorterClass=None,
flow=None):
"""
Set the root path and the sorter class used for sorting
The regex will serve (if we use validation) to validate that the filename has the following form:
SACN43_CWAO_012000_CYOJ_41613:ncp1:CWAO:SA:3.A.I.E::20050201200339"
FIXME: The regex should be passed otherwise! config file?
"""
# self.regex = re.compile(r'^[^:]*?:[^:]*?:[^:]*?:[^:]*?:(\d)[^:]*?:[^:]*?:(\d{14})$')
# This regex would be better than the one actually in use because it don't match the following entry:
# "A:B:C:WXO_DAN:BAD:3:ALLO:20050714183623"
# Surprisingly this entry match (priority in 6th position, should be in 5th) in spite of the use of *? operator ???
# Maybe we already accept some file badly named, so it's risky to change the regex now
self.regex = re.compile(r'^.*?:.*?:.*?:.*?:(\d).*?:.*?:(\d{14})$'
) # Regex used to validate filenames
self.path = path # Path from where we ingest filenames
self.flowName = os.path.basename(
path) # Last part of the path correspond to client/source name
self.validation = validation # Name Validation active (True or False)
self.patternMatching = patternMatching # Pattern matching active (True or False)
self.logger = logger # Use to log information
self.batch = batch # Maximum number of files that we are interested to sort
self.mtime = mtime # If we want to check modification time before taking a file
self.sortedFiles = [] # Sorted filenames
self.data = [
] # Content of x filenames (x is set in getFilesContent())
self.prioTree = prioTree # Boolean that determine if the "priorities" structure is enforced
self.sorterClass = sorterClass # Sorting algorithm that will be used by sort()
self.flow = flow # Flow (Client, Source, Sourlient) object, only used when patternMatching is True
maxEntries = 125000 # Flow (Client, Source, Sourlient) object, only used when patternMatching is True
if self.flow != None: maxEntries = self.flow.cache_size
self.cacheManager = CacheManager(maxEntries, timeout=12 *
3600) # Used to cache read entries
def test_load_mc(addr_list,
overwrite_memory=False,
ignore_lock=False,
force_set_valid=False):
''' Check (or redo) if all redis k-v are good.
- overwrite_memory: if you don't believe mm is loaded but flag shows valid, set this to true.
- ignore_lock: still load if other server already started loading.
- force_set_valid: overwrite valid_flags, regard as all loaded without checking. Do not use this option with other options. Run on 1 server.
'''
mm = CacheManager(overwrite_prefix=True,
overwrite_redis_servers=False,
rt_servers=False)
Pretend_Valid = force_set_valid # force change mm valid flag to 1. Careful! <set_to_be_valid>
tasks = []
for addr in addr_list:
metas = [
"osm/cache-%s-nodeid-to-lat-lng.txt" % addr, #0
"osm/cache-%s-nodeid-to-neighbor-nid.txt" % addr, #1
"osm/cache-%s-nids-to-speed.txt" % addr, #2
"osm/cache-%s-nid-to-elevation.txt" % addr, #3
"osm/cache-%s-nids-to-waytag.txt" % addr, #4
]
tasks.extend(metas)
lock = AccessRestrictionContext(
prefix="-test_load_mc~",
persistent_restriction=True,
persist_seconds=86400 * 3,
no_restriction=not On_Cluster,
)
for task in tasks:
if Pretend_Valid:
mm.use_cache(meta_file_name=task,
overwrite_prefix=True,
set_to_be_valid=True,
loading_msg=task)
continue
with lock:
lock.Access_Or_Skip(task)
print(task, MyIp)
mm.use_cache(meta_file_name=task,
overwrite_prefix=True,
overwrite_memory=overwrite_memory,
ignore_lock=ignore_lock,
loading_msg=task)
print("Done", addr_list, MyIp)
def __init__(self, ingestor, bullManager, reader ):
# General Attributes
self.bullManager = bullManager
self.ingestor = ingestor
self.logger = ingestor.logger
self.reader = reader
self.source = ingestor.source
self.files = []
self.data = []
self.entry = None
self.now = time.mktime(time.localtime())
# a cacheManager to make sure we process a file only once
self.cacheManager = CacheManager(maxEntries=self.source.cache_size, timeout=int(self.source.history)*3600)
# reading the collection station config file...
sp = StationParser.StationParser(PXPaths.STATION_TABLE)
sp.parse()
self.mapCollectionStation = sp.getStationsColl()
# instantiate the collection state class
self.collectionState = CollectionState.CollectionState( self )
self.getState = self.collectionState.getState
self.setState = self.collectionState.setState
self.updateState = self.collectionState.updateCollectionState
# instantiate the collection builder class
self.collectionBuilder = CollectionBuilder.CollectionBuilder( self )
self.collectionBuild = self.collectionBuilder.process
def __init__(self, client, logger):
self.client = client # Client object (give access to all configuration options)
self.timeout = client.timeout # No timeout for now
self.logger = logger # Logger object
self.connection = None # The connection
self.igniter = None
self.ssl = False
self.reader = DiskReader(PXPaths.TXQ + self.client.name, self.client.batch,
self.client.validation, self.client.patternMatching,
self.client.mtime, True, self.logger, eval(self.client.sorter), self.client)
self.debugFile = False
self.cacheManager = CacheManager(maxEntries=self.client.cache_size, timeout=8*3600)
# AMQP is there a max for message size
# self.set_maxLength(self.client.maxLength)
# statistics.
self.totBytes = 0
self.initialTime = time.time()
self.finalTime = None
self._connect()
class CollectionManager(object):
"""
The collectionManager reads RXQ dir, classifies the bulletins:
If they have to be ingested do it, if they have to be collected than collect and ingest.
Unrelated files are removed. Files that are not ready to be collected stay in the RXQ dir.
"""
def __init__(self, ingestor, bullManager, reader ):
# General Attributes
self.bullManager = bullManager
self.ingestor = ingestor
self.logger = ingestor.logger
self.reader = reader
self.source = ingestor.source
self.files = []
self.data = []
self.entry = None
self.now = time.mktime(time.localtime())
# a cacheManager to make sure we process a file only once
self.cacheManager = CacheManager(maxEntries=self.source.cache_size, timeout=int(self.source.history)*3600)
# reading the collection station config file...
sp = StationParser.StationParser(PXPaths.STATION_TABLE)
sp.parse()
self.mapCollectionStation = sp.getStationsColl()
# instantiate the collection state class
self.collectionState = CollectionState.CollectionState( self )
self.getState = self.collectionState.getState
self.setState = self.collectionState.setState
self.updateState = self.collectionState.updateCollectionState
# instantiate the collection builder class
self.collectionBuilder = CollectionBuilder.CollectionBuilder( self )
self.collectionBuild = self.collectionBuilder.process
#-----------------------------------------------------------------------------------------
# check if the bulletin is defined in the collection_station dictionnary
#-----------------------------------------------------------------------------------------
def conformWithStationDictionary( self ):
# working variables
dict = PXPaths.STATION_TABLE
dictkey = self.entry.dictkey
path = self.entry.path
station = self.entry.station
# check if header found in the dictionnary
if not dictkey in self.mapCollectionStation :
self.logger.warning("Reject %s : (%s) not in %s" % (path,dictkey,dict) )
self.unlink(path)
return False
# check if the station was found in bulletin
if station == None :
self.logger.warning("Reject %s : station %s not found" % (path,station) )
self.unlink(path)
return False
# check if the station is defined in the dictionnary
station_list = self.mapCollectionStation[dictkey]
if not station in station_list :
self.logger.warning("Reject %s : station %s not defined for %s in %s" % (path,station,dictkey,dict) )
self.unlink(path)
return False
return True
#-----------------------------------------------------------------------------------------
# check if the bulletin is defined in the collecteur's configuration
#-----------------------------------------------------------------------------------------
def conformWithSourceConfig( self ):
# working variables
bulltin = self.entry.bulletin
data = self.entry.data
path = self.entry.path
type = self.entry.type
BBB = self.entry.BBB
name = self.source.name
# get bulletin type and check if configured collectable, if not ignore
pos = -1
try : pos = self.source.headers.index(type)
except :
# if bulletin type not configured collectable ignore it
self.logger.warning("Reject %s : (%s) not define in %s.conf header" % (path,type,name) )
self.unlink(path)
return False
self.entry.sourceidx = pos
# get the bulletin emission and check if time ok
# specs says : if time not collectable or minute != 00 ingest NOW
hour = bulltin.emission[ 8:10]
minute = bulltin.emission[10:12]
if self.source.issue_hours[pos][0] != 'all' and not hour in self.source.issue_hours[pos] :
self.logger.info("Forced ingestion (time) %s" % path )
self.ingest(data)
self.unlink(path)
return False
if minute != '00' :
self.logger.info("Forced ingestion (time) %s" % path )
self.ingest(data)
self.unlink(path)
return False
# check if the bulletin is too early
history = 3600 * self.source.history
future = -60 * self.source.future
if bulltin.delay < future :
self.logger.warning("Reject %s : arrived earlier than permitted (%d)" % (path,bulltin.delay) )
self.unlink(path)
return False
# check if the bulletin is too old
if bulltin.delay > history :
self.logger.info("Forced ingestion (too old) %s" % path )
self.ingestX(self.entry,'Z')
self.unlink(path)
return False
# compute primary and cycle in secs
primary = 60 * int(self.source.issue_primary[pos])
cycle = 60 * int(self.source.issue_cycle[pos])
# if the bulletin is in its primary period
# do nothing if it is not time for collection
if bulltin.delay < primary :
if bulltin.age < primary :
self.logger.debug("File %s will be primary soon" % path )
return False
self.entry.period = 0
return True
# the bulletin is in one of its cycle period : compute the period and time of collection
self.entry.period = int((bulltin.delay - primary ) / cycle + 1)
timeOfCollection = primary + self.entry.period * cycle
# we can collect
if bulltin.age >= timeOfCollection : return True
# it's not time to collect and the bulletin is regular or a repeat: do nothing
if BBB == None or BBB[0] == 'R' :
self.logger.debug("File %s will be cycle soon" % path )
return False
# still not time to collect BUT AT THIS POINT
# we have an AMD or COR during one of its cycle... ingest !
return True
#-----------------------------------------------------------------------------------------
# ingest one bulletin's data
#-----------------------------------------------------------------------------------------
def ingest( self, data ):
self.bullManager.writeBulletinToDisk(data, True, True)
#-----------------------------------------------------------------------------------------
# ingestX : ingest one bulletin with a BBB having its last letter as X
#-----------------------------------------------------------------------------------------
def ingestX( self, entry, X ):
# working variables
bulltin = entry.bulletin
BBB = entry.BBB
data = entry.data
header = entry.header
# create its new BBB
cBBB = ''
if BBB == None : cBBB = 'RR' + X
elif BBB[0] == 'A' : cBBB = 'AA' + X
elif BBB[0] == 'C' : cBBB = 'CC' + X
else : cBBB = 'RR' + X
# rebuild data with new header
data = header[0] + ' ' + header[1] + ' ' + header[2] + ' ' + cBBB + '\n'
data += string.join(bulltin.bulletin[1:],'\n')
# ingest
self.ingest(data)
#-----------------------------------------------------------------------------------------
# collection process
#-----------------------------------------------------------------------------------------
def process( self ):
# setting current time
# it is very important to set this only once for the duration of the process
self.now = time.mktime(time.localtime())
# updating the collection state map
self.updateState(self.now)
# read it all
# NOTE : it is important not to restrict the reading
# so these lines don't use the batch option from config
self.reader.read()
self.logger.info("%d files in queue" % len(self.reader.sortedFiles))
# no files : call the collectionBuilder for empty primary collection...
if len(self.reader.sortedFiles) <= 0 :
self.collectionBuild(self.now)
return
# working variables
self.data = self.reader.getFilesContent()
self.files = self.reader.sortedFiles
# loop on all files
for index in range(len(self.data)):
# if the bulletin was already processed... skip it
if self.cacheManager.has( self.data[index] ) :
self.logger.info("File %s was cached earlier" % self.files[index] )
self.unlink(self.files[index])
continue
# bulletinCollection is a class to hold bulletin if it has to be collected
self.entry = CollectionEntry.CollectionEntry()
# generate bulletin, set its arrival, its age ... save it into a list
# TODO... if we have to set its arrival from clock than the file should
# be renamed with a few ":" and the arrival string date at the end
bulltin = bulletin.bulletin(self.data[index],self.logger)
try : bulltin.setArrivalStr(self.files[index].split(':')[6])
except : bulltin.setArrivalEp (self.now)
bulltin.compute_Age(self.now)
# check if bulletin and all its date are ok
if bulltin.errorBulletin != None or \
bulltin.arrival == None or bulltin.emission == None or \
bulltin.delay == None or bulltin.age == None or \
bulltin.ep_arrival == -1 or bulltin.ep_emission == -1 :
self.logger.warning("File %s had a problem...(most probably date)" % self.files[index] )
self.unlink(self.files[index])
continue
# put info in bulletinCollection entry
self.entry.path = self.files[index]
self.entry.data = self.data[index]
self.entry.bulletin = bulltin
self.entry.header = bulltin.getHeader().split()
self.entry.type = bulltin.getType()
self.entry.BBB = bulltin.getBBB()
self.entry.station = bulltin.getStation()
self.entry.dictkey = self.entry.header[0] + ' ' + self.entry.header[1]
self.entry.statekey = self.entry.header[0] + '_' + self.entry.header[1] + '_' + self.entry.header[2]
# info initialize and set later
self.entry.sourceidx = -1
self.entry.period = -1
# check if the bulletin is defined in the collection_station dictionnary
if not self.conformWithStationDictionary( ) : continue
# check if the bulletin is defined in the collecteur's configuration
if not self.conformWithSourceConfig( ) : continue
# check if the bulletin is not in conflict with its collection state
self.addToCollectionState( index )
# all files are classified... build collections if needed
# saving the collection state map if needed
self.collectionBuild(self.now)
#-----------------------------------------------------------------------------------------
# add the bulletin to the collection state map
#-----------------------------------------------------------------------------------------
def addToCollectionState( self, index ):
# working variables
path = self.entry.path
header = self.entry.header
BBB = self.entry.BBB
key = self.entry.statekey
# get MapCollectionState value
( period, amendement, correction, retard, Primary, Cycle ) = self.getState(key)
# the bulletin is not a primary
if self.entry.period > 0 :
Cycle.append(self.entry)
self.logger.debug("File %d %s : classified as Cycle" % (index,path) )
self.setState( key, period, amendement, correction, retard, Primary, Cycle )
return
# the bulletin is primary
# we have a problem if its primary collection was done...
# some possibilities : we were in recovery mode and some files were moved after a collecteur iteration
# we did not process enough files check the source.batch value
if period == 0 :
self.logger.warning("Reject %d %s : primary already done" % (index,path) )
self.unlink(path)
return
# primary bulletin are splitted...
# normal and repeated are place in Primary
# amendement and correction are place in Cycle
# the primary is collected and ingested first than
# the entries in cycle are ingested per period order 0,1,...
# this garanty that AMD and COR (even primary) are sent after
# the primary collection
if BBB == None :
Primary.append(self.entry)
self.logger.debug("File %d %s : classified as Primary" % (index,path) )
elif BBB[0] == 'R' :
Primary.append(self.entry)
self.logger.debug("File %d %s : classified as Primary" % (index,path) )
else :
Cycle.append(self.entry)
self.logger.debug("File %d %s : classified as Cycle " % (index,path) )
self.setState( key, period, amendement, correction, retard, Primary, Cycle )
#-----------------------------------------------------------------------------------------
# unlink a file
#-----------------------------------------------------------------------------------------
def unlink( self, path ):
try:
os.unlink(path)
self.logger.debug("%s has been erased", os.path.basename(path))
except OSError, e:
(type, value, tb) = sys.exc_info()
self.logger.error("Unable to unlink %s ! Type: %s, Value: %s" % (path, type, value))
def ingestSingleFile(self, igniter):
from DiskReader import DiskReader
from DirectRoutingParser import DirectRoutingParser
from PullFTP import PullFTP
if self.source.routemask:
self.drp = DirectRoutingParser(self.source.routingTable,
self.allNames, self.logger,
self.source.routing_version)
self.drp.parse()
if self.source.nodups:
self.fileCache = CacheManager(maxEntries=self.source.cache_size,
timeout=8 * 3600)
reader = DiskReader(self.ingestDir, self.source.batch,
self.source.validation,
self.source.patternMatching, self.source.mtime,
False, self.source.logger, self.source.sorter,
self.source)
sleep_sec = 1
if self.source.type == 'pull-file' or self.source.pull_script != None:
sleep_sec = self.source.pull_sleep
while True:
if igniter.reloadMode == True:
# We assign the defaults, reread configuration file for the source
# and reread all configuration file for the clients (all this in __init__)
if self.source.type == 'filter':
self.source.__init__(self.source.name, self.source.logger,
True, True)
else:
self.source.__init__(self.source.name, self.source.logger)
if self.source.routemask:
self.drp = DirectRoutingParser(self.source.routingTable,
self.allNames, self.logger)
self.drp.parse()
if self.source.nodups:
self.fileCache = CacheManager(
maxEntries=self.source.cache_size, timeout=8 * 3600)
reader = DiskReader(self.ingestDir, self.source.batch,
self.source.validation,
self.source.patternMatching,
self.source.mtime, False,
self.source.logger, self.source.sorter,
self.source)
self.logger.info("Receiver has been reloaded")
igniter.reloadMode = False
# pull files in rxq directory if in pull mode
if self.source.type == 'pull-file' or self.source.pull_script != None:
files = []
sleeping = os.path.isfile(PXPaths.RXQ + self.source.name +
'/.sleep')
if self.source.type == 'pull-file':
puller = PullFTP(self.source, self.logger, sleeping)
files = puller.get()
puller.close()
elif self.source.pull_script != None:
files = self.source.pull_script(self.source, self.logger,
sleeping)
if not sleeping:
self.logger.debug("Number of files pulled = %s" %
len(files))
else:
self.logger.info("This pull is sleeping")
# normal diskreader call for files
reader.read()
if len(reader.sortedFiles) <= 0:
time.sleep(sleep_sec)
continue
sortedFiles = reader.sortedFiles[:self.source.batch]
# processing the list if necessary...
if self.source.lx_execfile != None:
sfiles = []
sfiles.extend(sortedFiles)
self.logger.info("%d files process with lx_script" %
len(sfiles))
sortedFiles = self.source.run_lx_script(
sfiles, self.source.logger)
self.logger.info("%d files will be ingested" % len(sortedFiles))
for file in sortedFiles:
self.ingestFile(file)
def activeCache(self, cachedir):
"""キャッシュ保存先の設定とキャッシュ保存関数の有効化"""
self.cm = CacheManager(cachedir)
def activeCache(self, cachedir): """キャッシュ保存先の設定とキャッシュ保存関数の有効化""" self.cm = CacheManager(cachedir)
class AptP2P(protocol.Factory):
"""The main code object that does all of the work.
Contains all of the sub-components that do all the low-level work, and
coordinates communication between them.
@type dhtClass: L{interfaces.IDHT}
@ivar dhtClass: the DHT class to use
@type cache_dir: L{twisted.python.filepath.FilePath}
@ivar cache_dir: the directory to use for storing all files
@type db: L{db.DB}
@ivar db: the database to use for tracking files and hashes
@type dht: L{DHTManager.DHT}
@ivar dht: the manager for DHT requests
@type stats: L{stats.StatsLogger}
@ivar stats: the statistics logger to record sent data to
@type http_server: L{HTTPServer.TopLevel}
@ivar http_server: the web server that will handle all requests from apt
and from other peers
@type peers: L{PeerManager.PeerManager}
@ivar peers: the manager of all downloads from mirrors and other peers
@type mirrors: L{MirrorManager.MirrorManager}
@ivar mirrors: the manager of downloaded information about mirrors which
can be queried to get hashes from file names
@type cache: L{CacheManager.CacheManager}
@ivar cache: the manager of all downloaded files
@type my_addr: C{string}, C{int}
@ivar my_addr: the IP address and port of this peer
"""
def __init__(self, dhtClass):
"""Initialize all the sub-components.
@type dhtClass: L{interfaces.IDHT}
@param dhtClass: the DHT class to use
"""
log.msg('Initializing the main apt_p2p application')
self.dhtClass = dhtClass
self.my_addr = None
#{ Factory interface
def startFactory(self):
reactor.callLater(0, self._startFactory)
def _startFactory(self):
log.msg('Starting the main apt_p2p application')
self.cache_dir = FilePath(config.get('DEFAULT', 'CACHE_DIR'))
if not self.cache_dir.child(download_dir).exists():
self.cache_dir.child(download_dir).makedirs()
if not self.cache_dir.child(peer_dir).exists():
self.cache_dir.child(peer_dir).makedirs()
self.db = DB(self.cache_dir.child('apt-p2p.db'))
self.dht = DHT(self.dhtClass, self.db)
df = self.dht.start()
df.addCallback(self._dhtStarted)
self.stats = StatsLogger(self.db)
self.http_server = TopLevel(self.cache_dir.child(download_dir), self.db, self)
self.http_server.getHTTPFactory().startFactory()
self.peers = PeerManager(self.cache_dir.child(peer_dir), self.dht, self.stats)
self.mirrors = MirrorManager(self.cache_dir)
self.cache = CacheManager(self.cache_dir.child(download_dir), self.db, self)
def _dhtStarted(self, result):
"""Save the returned address and start scanning the cache."""
self.my_addr = result
self.cache.scanDirectories()
def stopFactory(self):
log.msg('Stoppping the main apt_p2p application')
self.http_server.getHTTPFactory().stopFactory()
self.mirrors.cleanup()
self.stats.save()
self.db.close()
def buildProtocol(self, addr):
return self.http_server.getHTTPFactory().buildProtocol(addr)
#{ Other functions
def getStats(self):
"""Retrieve and format the statistics for the program.
@rtype: C{string}
@return: the formatted HTML page containing the statistics
"""
out = '<html><body>\n\n'
out += self.stats.formatHTML(self.my_addr)
out += '\n\n'
out += self.dht.getStats()
out += '\n</body></html>\n'
return out
#{ Main workflow
def get_resp(self, req, url, orig_resp = None):
"""Lookup a hash for the file in the local mirror info.
Starts the process of getting a response to an apt request.
@type req: L{twisted.web2.http.Request}
@param req: the initial request sent to the HTTP server by apt
@param url: the URI of the actual mirror request
@type orig_resp: L{twisted.web2.http.Response}
@param orig_resp: the response from the cache to be sent to apt
(optional, ignored if missing)
@rtype: L{twisted.internet.defer.Deferred}
@return: a deferred that will be called back with the response
"""
d = defer.Deferred()
log.msg('Trying to find hash for %s' % url)
findDefer = self.mirrors.findHash(unquote(url))
findDefer.addCallbacks(self.findHash_done, self.findHash_error,
callbackArgs=(req, url, orig_resp, d),
errbackArgs=(req, url, orig_resp, d))
return d
def findHash_error(self, failure, req, url, orig_resp, d):
"""Process the error in hash lookup by returning an empty L{HashObject}."""
log.msg('Hash lookup for %s resulted in an error: %s' %
(url, failure.getErrorMessage()))
self.findHash_done(HashObject(), req, url, orig_resp, d)
def findHash_done(self, hash, req, url, orig_resp, d):
"""Use the returned hash to lookup the file in the cache.
If the hash was not found, the workflow skips down to download from
the mirror (L{startDownload}), or checks the freshness of an old
response if there is one.
@type hash: L{Hash.HashObject}
@param hash: the hash object containing the expected hash for the file
"""
if hash.expected() is None:
log.msg('Hash for %s was not found' % url)
# Send the old response or get a new one
if orig_resp:
self.check_freshness(req, url, orig_resp, d)
else:
self.startDownload([], req, hash, url, d)
else:
log.msg('Found hash %s for %s' % (hash.hexexpected(), url))
# Lookup hash in cache
locations = self.db.lookupHash(hash.expected(), filesOnly = True)
self.getCachedFile(hash, req, url, d, locations)
def check_freshness(self, req, url, orig_resp, d):
"""Send a HEAD to the mirror to check if the response from the cache is still valid.
@type req: L{twisted.web2.http.Request}
@param req: the initial request sent to the HTTP server by apt
@param url: the URI of the actual mirror request
@type orig_resp: L{twisted.web2.http.Response}
@param orig_resp: the response from the cache to be sent to apt
"""
log.msg('Checking if %s is still fresh' % url)
modtime = orig_resp.headers.getHeader('Last-Modified')
headDefer = self.peers.get(HashObject(), url, method = "HEAD",
modtime = modtime)
headDefer.addCallbacks(self.check_freshness_done,
self.check_freshness_error,
callbackArgs = (req, url, orig_resp, d),
errbackArgs = (req, url, d))
def check_freshness_done(self, resp, req, url, orig_resp, d):
"""Return the fresh response, if stale start to redownload.
@type resp: L{twisted.web2.http.Response}
@param resp: the response from the mirror to the HEAD request
@type req: L{twisted.web2.http.Request}
@param req: the initial request sent to the HTTP server by apt
@param url: the URI of the actual mirror request
@type orig_resp: L{twisted.web2.http.Response}
@param orig_resp: the response from the cache to be sent to apt
"""
if resp.code == 304:
log.msg('Still fresh, returning: %s' % url)
d.callback(orig_resp)
else:
log.msg('Stale, need to redownload: %s' % url)
self.startDownload([], req, HashObject(), url, d)
def check_freshness_error(self, err, req, url, d):
"""Mirror request failed, continue with download.
@param err: the response from the mirror to the HEAD request
@type req: L{twisted.web2.http.Request}
@param req: the initial request sent to the HTTP server by apt
@param url: the URI of the actual mirror request
"""
log.err(err)
self.startDownload([], req, HashObject(), url, d)
def getCachedFile(self, hash, req, url, d, locations):
"""Try to return the file from the cache, otherwise move on to a DHT lookup.
@type locations: C{list} of C{dictionary}
@param locations: the files in the cache that match the hash,
the dictionary contains a key 'path' whose value is a
L{twisted.python.filepath.FilePath} object for the file.
"""
if not locations:
log.msg('Failed to return file from cache: %s' % url)
self.lookupHash(req, hash, url, d)
return
# Get the first possible location from the list
file = locations.pop(0)['path']
log.msg('Returning cached file: %s' % file.path)
# Get it's response
resp = static.File(file.path).renderHTTP(req)
if isinstance(resp, defer.Deferred):
resp.addBoth(self._getCachedFile, hash, req, url, d, locations)
else:
self._getCachedFile(resp, hash, req, url, d, locations)
def _getCachedFile(self, resp, hash, req, url, d, locations):
"""Check the returned response to be sure it is valid."""
if isinstance(resp, failure.Failure):
log.msg('Got error trying to get cached file')
log.err(resp)
# Try the next possible location
self.getCachedFile(hash, req, url, d, locations)
return
log.msg('Cached response: %r' % resp)
if resp.code >= 200 and resp.code < 400:
d.callback(resp)
else:
# Try the next possible location
self.getCachedFile(hash, req, url, d, locations)
def lookupHash(self, req, hash, url, d):
"""Lookup the hash in the DHT."""
log.msg('Looking up hash in DHT for file: %s' % url)
key = hash.expected()
lookupDefer = self.dht.get(key)
lookupDefer.addBoth(self.startDownload, req, hash, url, d)
def startDownload(self, values, req, hash, url, d):
"""Start the download of the file.
The download will be from peers if the DHT lookup succeeded, or
from the mirror otherwise.
@type values: C{list} of C{dictionary}
@param values: the returned values from the DHT containing peer
download information
"""
# Remove some headers Apt sets in the request
req.headers.removeHeader('If-Modified-Since')
req.headers.removeHeader('Range')
req.headers.removeHeader('If-Range')
if not isinstance(values, list) or not values:
if not isinstance(values, list):
log.msg('DHT lookup for %s failed with error %r' % (url, values))
else:
log.msg('Peers for %s were not found' % url)
getDefer = self.peers.get(hash, url)
# getDefer.addErrback(self.final_fallback, hash, url)
getDefer.addCallback(self.cache.save_file, hash, url)
getDefer.addErrback(self.cache.save_error, url)
getDefer.addCallbacks(d.callback, d.errback)
else:
log.msg('Found peers for %s: %r' % (url, values))
# Download from the found peers
getDefer = self.peers.get(hash, url, values)
getDefer.addCallback(self.check_response, hash, url)
getDefer.addCallback(self.cache.save_file, hash, url)
getDefer.addErrback(self.cache.save_error, url)
getDefer.addCallbacks(d.callback, d.errback)
def check_response(self, response, hash, url):
"""Check the response from peers, and download from the mirror if it is not."""
if response.code < 200 or response.code >= 300:
log.msg('Download from peers failed, going to direct download: %s' % url)
getDefer = self.peers.get(hash, url)
# getDefer.addErrback(self.final_fallback, hash, url)
return getDefer
return response
def final_fallback(self, err, hash, url):
"""Final retry if the mirror still generated an error."""
log.msg('Download from mirror failed, retrying once only: %s' % url)
log.err(err)
getDefer = self.peers.get(hash, url)
return getDefer
def new_cached_file(self, file_path, hash, new_hash, url = None, forceDHT = False):
"""Add a newly cached file to the mirror info and/or the DHT.
If the file was downloaded, set url to the path it was downloaded for.
Doesn't add a file to the DHT unless a hash was found for it
(but does add it anyway if forceDHT is True).
@type file_path: L{twisted.python.filepath.FilePath}
@param file_path: the location of the file in the local cache
@type hash: L{Hash.HashObject}
@param hash: the original (expected) hash object containing also the
hash of the downloaded file
@type new_hash: C{boolean}
@param new_hash: whether the has was new to this peer, and so should
be added to the DHT
@type url: C{string}
@param url: the URI of the location of the file in the mirror
(optional, defaults to not adding the file to the mirror info)
@type forceDHT: C{boolean}
@param forceDHT: whether to force addition of the file to the DHT
even if the hash was not found in a mirror
(optional, defaults to False)
"""
if url:
self.mirrors.updatedFile(url, file_path)
if self.my_addr and hash and new_hash and (hash.expected() is not None or forceDHT):
return self.dht.store(hash)
return None
def ingestBulletinFile(self, igniter):
from DiskReader import DiskReader
import bulletinManager
import bulletinManagerAm
from PullFTP import PullFTP
sleep_sec = 1
if self.source.type == 'pull-bulletin' or self.source.pull_script != None:
sleep_sec = self.source.pull_sleep
bullManager = bulletinManager.bulletinManager(
self.ingestDir, self.logger, self.ingestDir, 99999, '\n',
self.source.extension, self.source.routingTable,
self.source.mapEnteteDelai, self.source,
self.source.addStationInFilename)
if self.source.bulletin_type == 'am':
bullManager = bulletinManagerAm.bulletinManagerAm(
self.ingestDir, self.logger, self.ingestDir, 99999, '\n',
self.source.extension, self.source.routingTable,
self.source.addSMHeader, PXPaths.STATION_TABLE,
self.source.mapEnteteDelai, self.source,
self.source.addStationInFilename)
if self.source.nodups:
self.fileCache = CacheManager(maxEntries=self.source.cache_size,
timeout=8 * 3600)
reader = DiskReader(bullManager.pathSource, self.source.batch,
self.source.validation,
self.source.patternMatching, self.source.mtime,
False, self.source.logger, self.source.sorter,
self.source)
while True:
# If a SIGHUP signal is received ...
if igniter.reloadMode == True:
# We assign the defaults, reread configuration file for the source
# and reread all configuration file for the clients (all this in __init__)
if self.source.type == 'filter-bulletin':
self.source.__init__(self.source.name, self.source.logger,
True, True)
else:
self.source.__init__(self.source.name, self.source.logger)
bullManager = bulletinManager.bulletinManager(
self.ingestDir, self.logger, self.ingestDir, 99999, '\n',
self.source.extension, self.source.routingTable,
self.source.mapEnteteDelai, self.source,
self.source.addStationInFilename)
if self.source.bulletin_type == 'am':
bullManager = bulletinManagerAm.bulletinManagerAm(
self.ingestDir, self.logger, self.ingestDir, 99999,
'\n', self.source.extension, self.source.routingTable,
self.source.addSMHeader, PXPaths.STATION_TABLE,
self.source.mapEnteteDelai, self.source,
self.source.addStationInFilename)
if self.source.nodups:
self.fileCache = CacheManager(
maxEntries=self.source.cache_size, timeout=8 * 3600)
reader = DiskReader(bullManager.pathSource, self.source.batch,
self.source.validation,
self.source.patternMatching,
self.source.mtime, False,
self.source.logger, self.source.sorter,
self.source)
self.logger.info("Receiver has been reloaded")
igniter.reloadMode = False
# pull files in rxq directory if in pull mode
if self.source.type == 'pull-bulletin' or self.source.pull_script != None:
files = []
sleeping = os.path.isfile(PXPaths.RXQ + self.source.name +
'/.sleep')
if self.source.type == 'pull-bulletin':
puller = PullFTP(self.source, self.logger, sleeping)
files = puller.get()
puller.close()
elif self.source.pull_script != None:
files = self.source.pull_script(self.source, self.logger,
sleeping)
if not sleeping:
self.logger.debug("Number of files pulled = %s" %
len(files))
else:
self.logger.info("This pull is sleeping")
# normal diskreader call for files
reader.read()
# processing the list if necessary...
if self.source.lx_execfile != None and len(reader.sortedFiles) > 0:
sfiles = []
sfiles.extend(reader.sortedFiles)
self.logger.info("%d files process with lx_script" %
len(sfiles))
sortedFiles = self.source.run_lx_script(
sfiles, self.source.logger)
reader.sortedFiles = sortedFiles
# continue normally
data = reader.getFilesContent(reader.batch)
if len(data) == 0:
time.sleep(sleep_sec)
continue
else:
self.logger.info("%d bulletins will be ingested", len(data))
# Write (and name correctly) the bulletins to disk, erase them after
for index in range(len(data)):
# ignore duplicate if requiered
duplicate = self.source.nodups and self.fileCache.find(
data[index], 'md5') is not None
#nb_bytes = len(data[index])
#self.logger.info("Lecture de %s: %d bytes" % (reader.sortedFiles[index], nb_bytes))
if not duplicate:
# converting the file if necessary
if self.source.fx_execfile != None:
file = reader.sortedFiles[index]
fxfile = self.source.run_fx_script(
file, self.source.logger)
# convertion did not work
if fxfile == None:
self.logger.warning(
"FX script ignored the file : %s" %
os.path.basename(file))
os.unlink(file)
continue
# file already in proper format
elif fxfile == file:
self.logger.warning(
"FX script kept the file as is : %s" %
os.path.basename(file))
# file converted...
else:
self.logger.info("FX script modified %s to %s " %
(os.path.basename(file),
os.path.basename(fxfile)))
os.unlink(file)
fp = open(fxfile, 'r')
dx = fp.read()
fp.close()
reader.sortedFiles[index] = fxfile
data[index] = dx
# writing/ingesting the bulletin
if isinstance(bullManager,
bulletinManagerAm.bulletinManagerAm):
bullManager.writeBulletinToDisk(data[index], True)
else:
bullManager.writeBulletinToDisk(
data[index], True, True)
try:
file = reader.sortedFiles[index]
os.unlink(file)
if duplicate:
self.logger.info("suppressed duplicate file %s",
os.path.basename(file))
self.logger.debug("%s has been erased",
os.path.basename(file))
except OSError, e:
(type, value, tb) = sys.exc_info()
self.logger.error(
"Unable to unlink %s ! Type: %s, Value: %s" %
(reader.sortedFiles[index], type, value))
class unittest_CacheManager(unittest.TestCase):
def setUp(self):
self.manager = CacheManager(maxEntries=3, timeout=5 * 3600)
self.files = [
'db/test-file-grib', 'db/test-file-burf', 'db/test-file-bulletin'
]
#f = open(self.files[0])
#self.data = f.read(1048576)
#while len(self.data) :
#print self.data
#self.data = f.read(1048576)
#f.close()
self.data = 'ceci est un test'
def test_CacheManager(self):
self.assertEqual(self.manager.get_md5_from_file(self.files[0]),
'57285445a1c80023b3f2e96546754d5b')
self.manager.find(self.data, 'md5')
self.manager.find(self.files[1])
#md5 of self.data = 11b35a0201513381dcdd130831f702d0
self.assertEqual(self.manager.has('11b35a0201513381dcdd130831f702d0'),
True)
self.assertEqual(self.manager.has(self.files[2]), False)
self.manager.find(self.data, 'md5')
self.manager.find(self.files[1])
self.manager.find(self.files[2])
self.assertEqual(self.manager.getStats(), ({1: 1, 2: 2}, 2.0, 5.0))
self.manager.find(self.files[0])
time.sleep(3)
self.manager.timeoutClear(3)
self.assertEqual(self.manager.cache, {})
def ingestSingleFile(self, igniter):
from DiskReader import DiskReader
from DirectRoutingParser import DirectRoutingParser
from PullFTP import PullFTP
if self.source.routemask :
self.drp = DirectRoutingParser(self.source.routingTable, self.allNames, self.logger, self.source.routing_version)
self.drp.parse()
if self.source.nodups :
self.fileCache = CacheManager(maxEntries=self.source.cache_size, timeout=8*3600)
reader = DiskReader(self.ingestDir, self.source.batch, self.source.validation, self.source.patternMatching,
self.source.mtime, False, self.source.logger, self.source.sorter, self.source)
sleep_sec = 1
if self.source.type == 'pull-file' or self.source.pull_script != None : sleep_sec = self.source.pull_sleep
while True:
if igniter.reloadMode == True:
# We assign the defaults, reread configuration file for the source
# and reread all configuration file for the clients (all this in __init__)
if self.source.type == 'filter' :
self.source.__init__(self.source.name, self.source.logger, True, True)
else :
self.source.__init__(self.source.name, self.source.logger)
if self.source.routemask :
self.drp = DirectRoutingParser(self.source.routingTable, self.allNames, self.logger)
self.drp.parse()
if self.source.nodups :
self.fileCache = CacheManager(maxEntries=self.source.cache_size, timeout=8*3600)
reader = DiskReader(self.ingestDir, self.source.batch, self.source.validation, self.source.patternMatching,
self.source.mtime, False, self.source.logger, self.source.sorter, self.source)
self.logger.info("Receiver has been reloaded")
igniter.reloadMode = False
# pull files in rxq directory if in pull mode
if self.source.type == 'pull-file' or self.source.pull_script != None :
files = []
sleeping = os.path.isfile(PXPaths.RXQ + self.source.name + '/.sleep')
if self.source.type == 'pull-file' :
puller = PullFTP(self.source,self.logger,sleeping)
files = puller.get()
puller.close()
elif self.source.pull_script != None :
files = self.source.pull_script(self.source,self.logger,sleeping)
if not sleeping :
self.logger.debug("Number of files pulled = %s" % len(files) )
else :
self.logger.info("This pull is sleeping")
# normal diskreader call for files
reader.read()
if len(reader.sortedFiles) <= 0:
time.sleep(sleep_sec)
continue
sortedFiles = reader.sortedFiles[:self.source.batch]
# processing the list if necessary...
if self.source.lx_execfile != None :
sfiles = []
sfiles.extend(sortedFiles)
self.logger.info("%d files process with lx_script" % len(sfiles))
sortedFiles = self.source.run_lx_script(sfiles,self.source.logger)
self.logger.info("%d files will be ingested" % len(sortedFiles))
for file in sortedFiles:
self.ingestFile(file)
addpath = mypydir + "../zyrcode/" # https://github.com/zyrgit/GreenRouteCode
if addpath not in sys.path: sys.path.append(addpath)
from mygmaps import GoogleMaps
from myosrm import gen_map_html_from_path_list
from common.osrm import get_fuel_given_latlng_list
addpath = mypydir + "../zyrcode/code/"
if addpath not in sys.path: sys.path.append(addpath)
from constants import * # addr2ip
''' Mapping green routes and google's to sumo net.
https://sumo.dlr.de/wiki/Tools/Routes#tracemapper.py
'''
gmaps = GoogleMaps() # need your API key!
_Cache_Dir = "cache/"
mm_nid2latlng = CacheManager(overwrite_prefix=True) # need redis
def gen_edges_tracemapper(traceFile, netFile, routeFile):
cmd = SUMO_Tools_dir + "/route/tracemapper.py -v --geo --fill-gaps --delta 10 -n %s -t %s -o %s" % (
netFile, traceFile, routeFile)
print("\n" + cmd)
subprocess.call(cmd, shell=True)
def fix_edges_routecheck(netFile, routeFile):
cmd = SUMO_Tools_dir + "/route/routecheck.py -v --fix --inplace --net %s %s" % (
netFile, routeFile)
print("\n" + cmd)
subprocess.call(cmd, shell=True)
class Ingestor(object):
"""
Normally, an Ingestor will be in a Source. It can also be used for the only reason that this object has
access to all the configuration options of the clients. For this particular case, source=None.
"""
def __init__(self, source=None, logger=None):
# General Attributes
self.source = source
self.reader = None
self.drp = None
self.count = 0
self.Mcount = 99999
if source is not None:
self.ingestDir = PXPaths.RXQ + self.source.name
if self.source.type == 'filter' or self.source.type == 'filter-bulletin' :
self.ingestDir = PXPaths.FXQ + self.source.name
self.logger = source.logger
elif logger is not None:
self.logger = logger
self.pxManager = PXManager() # Create a manager
self.pxManager.setLogger(self.logger) # Give the logger to the the manager
self.pxManager.initNames() # Set rx and tx names
self.clientNames = self.pxManager.getTxNames() # Obtains the list of client's names (the ones to wich we can link files)
self.filterNames = self.pxManager.getFxNames() # Obtains the list of filter's names (the ones to wich we can link files)
if source is not None:
if self.source.name in self.filterNames : self.filterNames.remove(self.source.name)
self.sourlientNames = self.pxManager.getTRxNames() # Obtains the list of sourlient's names (the ones to wich we can link files)
self.allNames = self.clientNames + self.filterNames + self.sourlientNames # Clients + Sourlients names
self.clients = {} # All the Client/Filter/Sourlient objects
self.fileCache = None # product processed.
self.dbDirsCache = CacheManager(maxEntries=200000, timeout=25*3600) # Directories created in the DB
self.clientDirsCache = CacheManager(maxEntries=25000, timeout=2*3600) # File ingested in RXQ
self.feedNames = [] # source to feed
self.feeds = {}
if source is not None:
self.logger.info("Ingestor (source %s) can link files to clients: %s" % (source.name, self.allNames))
def setFeeds(self, feedNames ):
from Source import Source
sources = self.pxManager.getRxNames()
for name in feedNames :
if not name in sources : continue
instant = Source(name, self.logger, False)
if instant.type == 'am' or instant.type == 'amqp' or instant.type == 'wmo' :
self.logger.warning("Feed (source %s) will be ignored (type %s)" % (name, instant.type) )
continue
self.feedNames.append(name)
self.feeds[name] = instant
self.logger.info("Ingestor (source %s) can link files to receiver: %s" % (self.source.name, self.feedNames))
def createDir(self, dir, cacheManager):
if cacheManager.find(dir) == None:
try:
os.makedirs(dir, 01775)
except OSError:
(type, value, tb) = sys.exc_info()
self.logger.debug("Problem when creating dir (%s) => Type: %s, Value: %s" % (dir, type, value))
def setClients(self):
""""
Set a dictionnary of Clients. Main usage will be to access value of
configuration options (mainly masks) of the Client objects.
"""
from Source import Source
from Sourlient import Sourlient
for name in self.clientNames:
self.clients[name] = Client(name, self.logger)
for name in self.filterNames :
self.clients[name] = Source(name, self.logger, False, True)
for name in self.sourlientNames:
self.clients[name] = Sourlient(name, self.logger, False)
#self.clients[name].readConfig()
#print self.clients[name].masks
def getIngestName(self, receptionName):
"""
Map reception name to ingest name, based on the source configuration.
This just inserts missing fields, like whattopds. DUMB!
FIXME: Have a library of functions, configurable per source, to
perform the mapping, perhaps using rmasks ? & other args.
"""
receptionNameParts = receptionName.split(':')
extensionParts = self.source.extension.split(':')
if len(receptionNameParts) > 6 :
receptionNameParts = receptionNameParts[:6]
self.logger.warning("File %s truncated to %s" % (receptionName,':'.join(receptionNameParts) ) )
for i in range(1,6):
if len(receptionNameParts) == i :
receptionNameParts = receptionNameParts + [extensionParts[i]]
elif receptionNameParts[i] == '':
receptionNameParts[i] = extensionParts[i]
receptionNameParts = receptionNameParts + [time.strftime("%Y%m%d%H%M%S", time.gmtime())]
return string.join(receptionNameParts,':')
def getClientQueueName(self, clientName, ingestName, priority=None):
"""
Return the directory into which a file of a given priority should be placed.
Layout used is: /apps/px/txq/<client>/<priority>/YYYYmmddhh
"""
parts = ingestName.split(':')
if not priority:
priority = parts[4].split('.')[0]
clientpathName = PXPaths.TXQ + clientName + '/' + str(priority) + '/' + time.strftime("%Y%m%d%H", time.gmtime()) + '/' + ingestName
if clientName in self.filterNames :
clientpathName = PXPaths.FXQ + clientName + '/' + ingestName
return clientpathName
def getDBName(self, ingestName):
"""
Given an ingest name, return a relative database name
Given a file name of the form:
what : ori_system : ori_site : data_type : format :
link it to:
db/<today>/data_type/ori_system/ori_site/ingestName
(same pattern as PDS)
NB: see notes/tests.txt for why the date/time is recalculated everytime.
"""
if ingestName.count(':') >= 4:
today = time.strftime("%Y%m%d", time.gmtime())
dirs = ingestName.split(':')
return PXPaths.DB + today + '/' + dirs[3] + '/' + dirs[1] + '/' + dirs[2] + '/' + ingestName
else:
return ''
def getRouteKey(self, filename):
"""
Given an ingest name, return a route key based on the imask given
"""
# check against the masks
for mask in self.source.masks:
# no match
if not mask[3].match(filename) : continue
# reject
if not mask[4] : return None
# accept... so key generation
parts = re.findall( mask[0], filename )
if len(parts) == 2 and parts[1] == '' : parts.pop(1)
if len(parts) != 1 : continue
key = parts[0]
if isinstance(parts[0],tuple) : key = '_'.join(parts[0])
self.logger.debug("RouteKey Key = %s Mask = %s Filename = %s" % (key,mask[0],filename) )
return key
# fallback behavior return filename
return filename
def isMatching(self, client, ingestName):
"""
Verify if ingestName is matching one mask of a client
"""
from Source import Source
if len(client.masks_deprecated) > 0 :
for mask in client.masks_deprecated:
if fnmatch.fnmatch(ingestName, mask[0]):
try:
if mask[2]:
return True
except:
return False
for mask in client.masks:
if mask[3].match(ingestName ) : return mask[4]
if isinstance(client,Source) : return True
return False
def getMatchingClientNamesFromMasks(self, ingestName, potentialClientNames):
matchingClientNames = []
for name in potentialClientNames:
try:
if self.isMatching(self.clients[name], ingestName):
matchingClientNames.append(name)
except KeyError:
pass
return matchingClientNames
def getMatchingFeedNamesFromMasks(self, ingestName, potentialFeedNames):
matchingFeedNames = []
for name in potentialFeedNames:
try:
if self.feeds[name].fileMatchMask(ingestName):
matchingFeedNames.append(name)
except KeyError:
pass
return matchingFeedNames
def ingest(self, receptionName, ingestName, clientNames, priority=None ):
self.logger.debug("Reception Name: %s" % receptionName)
dbName = self.getDBName(ingestName)
if dbName == '':
self.logger.warning('Bad ingest name (%s) => No dbName' % ingestName)
return 0
self.createDir(os.path.dirname(dbName), self.dbDirsCache)
#try:
# os.link(receptionName, dbName)
#except OSError:
# (type, value, tb) = sys.exc_info()
# self.logger.error("Unable to link %s %s, Type: %s, Value: %s" % (receptionName, dbName, type, value))
os.link(receptionName, dbName)
nbBytes = os.stat(receptionName)[stat.ST_SIZE]
if self.source.debug:
self.logger.info("DBDirsCache: %s" % self.dbDirsCache.cache)
stats, cached, total = self.dbDirsCache.getStats()
if total:
percentage = "%2.2f %% of the last %i requests were cached" % (cached/total * 100, total)
else:
percentage = "No entries in the cache"
self.logger.info("DB Caching stats: %s => %s" % (str(stats), percentage))
self.logger.debug("ClientDirsCache: %s" % self.clientDirsCache.cache)
stats, cached, total = self.clientDirsCache.getStats()
if total:
percentage = "%2.2f %% of the last %i requests were cached" % (cached/total * 100, total)
else:
percentage = "No entries in the cache"
self.logger.debug("Client Caching stats: %s => %s" % (str(stats), percentage))
self.logger.info("Ingestion Name: %s" % ingestName)
self.logger.info("(%i Bytes) Ingested in DB as %s" % (nbBytes, dbName))
# Problem bulletins are databased, but not sent to clients
if ingestName.find("PROBLEM_BULLETIN") is not -1:
return 1
for name in clientNames:
clientQueueName = self.getClientQueueName(name, ingestName, priority)
self.createDir(os.path.dirname(clientQueueName), self.clientDirsCache)
#try:
# os.link(dbName, clientQueueName)
#except OSError:
# (type, value, tb) = sys.exc_info()
# self.logger.error("Unable to link %s %s, Type: %s, Value: %s" % (dbName, clientQueueName, type, value))
os.link(dbName, clientQueueName)
feedNames = []
if len(self.feedNames) > 0 :
feedNames = self.getMatchingFeedNamesFromMasks(ingestName, self.feedNames )
self.logger.debug("Matching (from patterns) feed names: %s" % feedNames )
for name in feedNames:
if name in clientNames : continue
sourceQueueName = PXPaths.RXQ + name + '/' + ingestName
self.createDir(os.path.dirname(sourceQueueName), self.clientDirsCache)
os.link(dbName, sourceQueueName)
self.logger.info("Queued for: %s" % string.join(clientNames) + ' ' + string.join(feedNames) )
return 1
def run(self):
if self.source.type == 'single-file' or self.source.type == 'pull-file':
self.ingestSingleFile()
elif self.source.type == 'bulletin-file' or self.source.type == 'pull-bulletin':
self.ingestBulletinFile()
elif self.source.type == 'collector':
self.ingestCollection()
def ingestSingleFile(self, igniter):
from DiskReader import DiskReader
from DirectRoutingParser import DirectRoutingParser
from PullFTP import PullFTP
if self.source.routemask :
self.drp = DirectRoutingParser(self.source.routingTable, self.allNames, self.logger, self.source.routing_version)
self.drp.parse()
if self.source.nodups :
self.fileCache = CacheManager(maxEntries=self.source.cache_size, timeout=8*3600)
reader = DiskReader(self.ingestDir, self.source.batch, self.source.validation, self.source.patternMatching,
self.source.mtime, False, self.source.logger, self.source.sorter, self.source)
sleep_sec = 1
if self.source.type == 'pull-file' or self.source.pull_script != None : sleep_sec = self.source.pull_sleep
while True:
if igniter.reloadMode == True:
# We assign the defaults, reread configuration file for the source
# and reread all configuration file for the clients (all this in __init__)
if self.source.type == 'filter' :
self.source.__init__(self.source.name, self.source.logger, True, True)
else :
self.source.__init__(self.source.name, self.source.logger)
if self.source.routemask :
self.drp = DirectRoutingParser(self.source.routingTable, self.allNames, self.logger)
self.drp.parse()
if self.source.nodups :
self.fileCache = CacheManager(maxEntries=self.source.cache_size, timeout=8*3600)
reader = DiskReader(self.ingestDir, self.source.batch, self.source.validation, self.source.patternMatching,
self.source.mtime, False, self.source.logger, self.source.sorter, self.source)
self.logger.info("Receiver has been reloaded")
igniter.reloadMode = False
# pull files in rxq directory if in pull mode
if self.source.type == 'pull-file' or self.source.pull_script != None :
files = []
sleeping = os.path.isfile(PXPaths.RXQ + self.source.name + '/.sleep')
if self.source.type == 'pull-file' :
puller = PullFTP(self.source,self.logger,sleeping)
files = puller.get()
puller.close()
elif self.source.pull_script != None :
files = self.source.pull_script(self.source,self.logger,sleeping)
if not sleeping :
self.logger.debug("Number of files pulled = %s" % len(files) )
else :
self.logger.info("This pull is sleeping")
# normal diskreader call for files
reader.read()
if len(reader.sortedFiles) <= 0:
time.sleep(sleep_sec)
continue
sortedFiles = reader.sortedFiles[:self.source.batch]
# processing the list if necessary...
if self.source.lx_execfile != None :
sfiles = []
sfiles.extend(sortedFiles)
self.logger.info("%d files process with lx_script" % len(sfiles))
sortedFiles = self.source.run_lx_script(sfiles,self.source.logger)
self.logger.info("%d files will be ingested" % len(sortedFiles))
for file in sortedFiles:
self.ingestFile(file)
def ingestFile(self, file ):
# check for duplicated if user requieres
if self.source.nodups :
# get md5 key from file...
md5_key = self.fileCache.get_md5_from_file(file)
# If data is already in cache, we don't send it
if self.fileCache.find(md5_key, 'standard') is not None:
os.unlink(file)
self.logger.info("suppressed duplicate file %s", os.path.basename(file))
return
# converting the file if necessary
if self.source.fx_execfile != None :
fxfile = self.source.run_fx_script(file,self.source.logger)
# convertion did not work
if fxfile == None :
self.logger.warning("FX script ignored the file : %s" % os.path.basename(file) )
os.unlink(file)
return
# file already in proper format
elif fxfile == file :
self.logger.warning("FX script kept the file as is : %s" % os.path.basename(file) )
# file converted...
else :
self.logger.info("FX script modified %s to %s " % (os.path.basename(file),os.path.basename(fxfile)) )
os.unlink(file)
file = fxfile
# filename to ingest
ingestName = self.getIngestName(os.path.basename(file))
# make sure we do not have a dbName already in there
# if it happens add a second to the postfix datetime stamp
# until the dbName is not found
dbName = self.getDBName(ingestName)
if dbName != '' and os.path.exists(dbName) :
ingestOrig = ingestName
self.logger.warning('dbName %s found in db, attempt to modify suffix ' % ingestName)
sec = int(ingestName[-2:]) + 1
while( sec <= 59 ) :
ingestName = ingestName[:-2] + "%.2d" % sec
dbName = self.getDBName(ingestName)
if not os.path.exists(dbName) : break
sec = sec + 1
# our trick did not work... process will bumb
if sec == 60 : ingestName = ingestOrig
# usual clients
potentials = self.clientNames + self.filterNames
# routing clients and priority (accept mask + routing info)
priority = None
if self.source.routemask :
# ingestBase is the ingestName without the postfix reception date
lst = ingestName.split(':')
pos = -1
if lst[-2] == '' : pos = -2
ingestBase = ':'.join(lst[:pos])
# build the key
key = self.getRouteKey(ingestBase)
# get the clients for that key (if possible)
lst = None
potentials = []
if key != None :
lst = self.drp.getClients(key)
if lst != None :
potentials = lst
priority = self.drp.getHeaderPriority(key)
else :
self.logger.warning("Key not in routing table (%s from %s)" % (key,ingestName) )
else :
self.logger.warning("Key not generated (no accept match) with %s" % ingestName )
# ingesting the file
matchingClients = self.getMatchingClientNamesFromMasks(ingestName, potentials )
self.logger.debug("Matching (from patterns) client names: %s" % matchingClients)
self.ingest(file, ingestName, matchingClients, priority )
os.unlink(file)
def ingestBulletinFile(self, igniter):
from DiskReader import DiskReader
import bulletinManager
import bulletinManagerAm
from PullFTP import PullFTP
sleep_sec = 1
if self.source.type == 'pull-bulletin' or self.source.pull_script != None : sleep_sec = self.source.pull_sleep
bullManager = bulletinManager.bulletinManager(
self.ingestDir,
self.logger,
self.ingestDir,
99999,
'\n',
self.source.extension,
self.source.routingTable,
self.source.mapEnteteDelai,
self.source,
self.source.addStationInFilename)
if self.source.bulletin_type == 'am' :
bullManager = bulletinManagerAm.bulletinManagerAm(
self.ingestDir,
self.logger,
self.ingestDir,
99999,
'\n',
self.source.extension,
self.source.routingTable,
self.source.addSMHeader,
PXPaths.STATION_TABLE,
self.source.mapEnteteDelai,
self.source,
self.source.addStationInFilename)
if self.source.nodups :
self.fileCache = CacheManager(maxEntries=self.source.cache_size, timeout=8*3600)
reader = DiskReader(bullManager.pathSource, self.source.batch, self.source.validation, self.source.patternMatching,
self.source.mtime, False, self.source.logger, self.source.sorter, self.source)
while True:
# If a SIGHUP signal is received ...
if igniter.reloadMode == True:
# We assign the defaults, reread configuration file for the source
# and reread all configuration file for the clients (all this in __init__)
if self.source.type == 'filter-bulletin' :
self.source.__init__(self.source.name, self.source.logger, True, True)
else :
self.source.__init__(self.source.name, self.source.logger)
bullManager = bulletinManager.bulletinManager(
self.ingestDir,
self.logger,
self.ingestDir,
99999,
'\n',
self.source.extension,
self.source.routingTable,
self.source.mapEnteteDelai,
self.source,
self.source.addStationInFilename)
if self.source.bulletin_type == 'am' :
bullManager = bulletinManagerAm.bulletinManagerAm(
self.ingestDir,
self.logger,
self.ingestDir,
99999,
'\n',
self.source.extension,
self.source.routingTable,
self.source.addSMHeader,
PXPaths.STATION_TABLE,
self.source.mapEnteteDelai,
self.source,
self.source.addStationInFilename)
if self.source.nodups :
self.fileCache = CacheManager(maxEntries=self.source.cache_size, timeout=8*3600)
reader = DiskReader(bullManager.pathSource, self.source.batch, self.source.validation, self.source.patternMatching,
self.source.mtime, False, self.source.logger, self.source.sorter,self.source)
self.logger.info("Receiver has been reloaded")
igniter.reloadMode = False
# pull files in rxq directory if in pull mode
if self.source.type == 'pull-bulletin' or self.source.pull_script != None :
files = []
sleeping = os.path.isfile(PXPaths.RXQ + self.source.name + '/.sleep')
if self.source.type == 'pull-bulletin' :
puller = PullFTP(self.source,self.logger,sleeping)
files = puller.get()
puller.close()
elif self.source.pull_script != None :
files = self.source.pull_script(self.source,self.logger,sleeping)
if not sleeping :
self.logger.debug("Number of files pulled = %s" % len(files) )
else :
self.logger.info("This pull is sleeping")
# normal diskreader call for files
reader.read()
# processing the list if necessary...
if self.source.lx_execfile != None and len(reader.sortedFiles) > 0:
sfiles = []
sfiles.extend(reader.sortedFiles)
self.logger.info("%d files process with lx_script" % len(sfiles))
sortedFiles = self.source.run_lx_script(sfiles,self.source.logger)
reader.sortedFiles = sortedFiles
# continue normally
data = reader.getFilesContent(reader.batch)
if len(data) == 0:
time.sleep(sleep_sec)
continue
else:
self.logger.info("%d bulletins will be ingested", len(data))
# Write (and name correctly) the bulletins to disk, erase them after
for index in range(len(data)):
# ignore duplicate if requiered
duplicate = self.source.nodups and self.fileCache.find(data[index], 'md5') is not None
#nb_bytes = len(data[index])
#self.logger.info("Lecture de %s: %d bytes" % (reader.sortedFiles[index], nb_bytes))
if not duplicate :
# converting the file if necessary
if self.source.fx_execfile != None :
file = reader.sortedFiles[index]
fxfile = self.source.run_fx_script(file,self.source.logger)
# convertion did not work
if fxfile == None :
self.logger.warning("FX script ignored the file : %s" % os.path.basename(file) )
os.unlink(file)
continue
# file already in proper format
elif fxfile == file :
self.logger.warning("FX script kept the file as is : %s" % os.path.basename(file) )
# file converted...
else :
self.logger.info("FX script modified %s to %s " % (os.path.basename(file),os.path.basename(fxfile)) )
os.unlink(file)
fp = open(fxfile,'r')
dx = fp.read()
fp.close()
reader.sortedFiles[index] = fxfile
data[index] = dx
# writing/ingesting the bulletin
if isinstance(bullManager,bulletinManagerAm.bulletinManagerAm):
bullManager.writeBulletinToDisk(data[index], True)
else :
bullManager.writeBulletinToDisk(data[index], True, True)
try:
file = reader.sortedFiles[index]
os.unlink(file)
if duplicate : self.logger.info("suppressed duplicate file %s", os.path.basename(file))
self.logger.debug("%s has been erased", os.path.basename(file))
except OSError, e:
(type, value, tb) = sys.exc_info()
self.logger.error("Unable to unlink %s ! Type: %s, Value: %s" % (reader.sortedFiles[index], type, value))
class WebSearcher:
def __init__(self):
self.cm = False
self.sleeptime = 2 #キャッシュ生成後にsleepする秒数
self.swWrited = False #検索結果のキャッシュを強制的に上書きするか(swWritedがTrueの場合に有効)
self.__params = {}
def activeCache(self, cachedir):
"""キャッシュ保存先の設定とキャッシュ保存関数の有効化"""
self.cm = CacheManager(cachedir)
def getHitCount(self):
"""get the number of retrieved pages
@return int the number of retrieved pages
"""
hits = False
if self.__engine == "tsubaki":
url = "http://tsubaki.ixnlp.nii.ac.jp/api.cgi?query=%s&only_hitcount=1" % self.__query
force_dpnd = self.getParameter("force_dpnd")
if force_dpnd:
url += "&force_dpnd=" + str(force_dpnd)
print url
if self.is_available_caching():
res = self.runCacheFunc(url)
else:
res = urllib.urlopen(url).read()
if not re.match("^[0-9]+$", res):
return "error"
hits = res.rstrip()
elif self.__engine == "yahoo" or self.__engine == "yahoo2":
if self.is_available_caching():
xmlstr = self.runCacheFunc(self.getResultURI(1, 1))
print xmlstr
doc = ElementTree(fromstring(xmlstr))
else:
url = self.getResultURI(1, 1)
fd = file(url, "rb")
doc = ElementTree(file=fd)
e = doc.getroot()
hits = e.attrib["totalResultsAvailable"]
return hits
def getResultURI(self, start, num):
engine = self.getEngine()
if engine == "tsubaki":
url = "http://tsubaki.ixnlp.nii.ac.jp/api.cgi?query=" + self.getQuery(
) + "&start=" + str(start) + "&results=" + str(num) + "&Snippet=1"
elif engine == "yahoo":
appid = self.getParameter("appid")
if not appid:
sys.exit("You must assign appid!!")
type = self.getParameter("type")
if not type:
type = "all"
url = "http://search.yahooapis.jp/WebSearchService/V1/webSearch?query=%s&appid=%s&type=%s&start=%s&results=%s&format=any" % (
self.__query, appid, type, start, num)
print url
elif engine == "yahoo2":
appid = self.getParameter("appid")
if not appid:
sys.exit("You must assign appid!!")
type = self.getParameter("type")
if not type:
type = "all"
url = "http://search.yahooapis.jp/WebSearchService/V2/webSearch?query=%s&appid=%s&type=%s&start=%s&results=%s&format=html" % (
self.__query, appid, type, start, num)
else:
url = False
return url
#"?query=" + urlencode(fnkf("-w", $this->query)) \
def is_available(self, engine):
"""check which defined search engine is available"""
available_engines = ["okwave", "tsubaki", "yahoo", "yahooQA"]
return engine in available_engines
def is_available_caching(self):
return self.cm != False
def runCacheFunc(self, url):
return self.cm.rwCache(url, self.sleeptime, self.swWrited)
def search(self, start, max):
"""run searching Web"""
engine = self.getEngine()
result = {}
rank = 1
if engine == "tsubaki":
url = self.getResultURI(start, max)
if self.is_available_caching():
xmlstr = self.runCacheFunc(url)
doc = ElementTree(fromstring(xmlstr))
else:
fd = file(url, "rb")
doc = ElementTree(file=fd)
for e in doc.findall("//Result"):
tmp = {
"title":
e.find("Title").text,
"url":
e.find("Url").text,
"snippet":
e.find("Snippet").text,
"cache":
"http://tsubaki.ixnlp.nii.ac.jp/api.cgi?id=" +
e.attrib["Id"] + "&format=html"
}
result[rank] = tmp
rank += 1
return result
def getEngine(self):
return self.__engine
def setEngine(self, engine):
self.__engine = engine
def getParameter(self, name):
if self.__params.has_key(name):
return self.__params[name]
else:
return False
def setParameter(self, name, value):
self.__params[name] = value
def getQuery(self):
return self.__query
def setQuery(self, query):
self.__query = query
## プロパティ ##
engine = property(getEngine, setEngine)
query = property(getQuery, setQuery)
class AptP2P(protocol.Factory):
"""The main code object that does all of the work.
Contains all of the sub-components that do all the low-level work, and
coordinates communication between them.
@type dhtClass: L{interfaces.IDHT}
@ivar dhtClass: the DHT class to use
@type cache_dir: L{twisted.python.filepath.FilePath}
@ivar cache_dir: the directory to use for storing all files
@type db: L{db.DB}
@ivar db: the database to use for tracking files and hashes
@type dht: L{DHTManager.DHT}
@ivar dht: the manager for DHT requests
@type stats: L{stats.StatsLogger}
@ivar stats: the statistics logger to record sent data to
@type http_server: L{HTTPServer.TopLevel}
@ivar http_server: the web server that will handle all requests from apt
and from other peers
@type peers: L{PeerManager.PeerManager}
@ivar peers: the manager of all downloads from mirrors and other peers
@type mirrors: L{MirrorManager.MirrorManager}
@ivar mirrors: the manager of downloaded information about mirrors which
can be queried to get hashes from file names
@type cache: L{CacheManager.CacheManager}
@ivar cache: the manager of all downloaded files
@type my_addr: C{string}, C{int}
@ivar my_addr: the IP address and port of this peer
"""
def __init__(self, dhtClass):
"""Initialize all the sub-components.
@type dhtClass: L{interfaces.IDHT}
@param dhtClass: the DHT class to use
"""
log.msg('Initializing the main apt_p2p application')
self.dhtClass = dhtClass
self.my_addr = None
#{ Factory interface
def startFactory(self):
reactor.callLater(0, self._startFactory)
def _startFactory(self):
log.msg('Starting the main apt_p2p application')
self.cache_dir = FilePath(config.get('DEFAULT', 'CACHE_DIR'))
if not self.cache_dir.child(download_dir).exists():
self.cache_dir.child(download_dir).makedirs()
if not self.cache_dir.child(peer_dir).exists():
self.cache_dir.child(peer_dir).makedirs()
self.db = DB(self.cache_dir.child('apt-p2p.db'))
self.dht = DHT(self.dhtClass, self.db)
df = self.dht.start()
df.addCallback(self._dhtStarted)
self.stats = StatsLogger(self.db)
self.http_server = TopLevel(self.cache_dir.child(download_dir),
self.db, self)
self.http_server.getHTTPFactory().startFactory()
self.peers = PeerManager(self.cache_dir.child(peer_dir), self.dht,
self.stats)
self.mirrors = MirrorManager(self.cache_dir)
self.cache = CacheManager(self.cache_dir.child(download_dir), self.db,
self)
def _dhtStarted(self, result):
"""Save the returned address and start scanning the cache."""
self.my_addr = result
self.cache.scanDirectories()
def stopFactory(self):
log.msg('Stoppping the main apt_p2p application')
self.http_server.getHTTPFactory().stopFactory()
self.mirrors.cleanup()
self.stats.save()
self.db.close()
def buildProtocol(self, addr):
return self.http_server.getHTTPFactory().buildProtocol(addr)
#{ Other functions
def getStats(self):
"""Retrieve and format the statistics for the program.
@rtype: C{string}
@return: the formatted HTML page containing the statistics
"""
out = '<html><body>\n\n'
out += self.stats.formatHTML(self.my_addr)
out += '\n\n'
out += self.dht.getStats()
out += '\n</body></html>\n'
return out
#{ Main workflow
def get_resp(self, req, url, orig_resp=None):
"""Lookup a hash for the file in the local mirror info.
Starts the process of getting a response to an apt request.
@type req: L{twisted.web2.http.Request}
@param req: the initial request sent to the HTTP server by apt
@param url: the URI of the actual mirror request
@type orig_resp: L{twisted.web2.http.Response}
@param orig_resp: the response from the cache to be sent to apt
(optional, ignored if missing)
@rtype: L{twisted.internet.defer.Deferred}
@return: a deferred that will be called back with the response
"""
d = defer.Deferred()
log.msg('Trying to find hash for %s' % url)
findDefer = self.mirrors.findHash(unquote(url))
findDefer.addCallbacks(self.findHash_done,
self.findHash_error,
callbackArgs=(req, url, orig_resp, d),
errbackArgs=(req, url, orig_resp, d))
return d
def findHash_error(self, failure, req, url, orig_resp, d):
"""Process the error in hash lookup by returning an empty L{HashObject}."""
log.msg('Hash lookup for %s resulted in an error: %s' %
(url, failure.getErrorMessage()))
self.findHash_done(HashObject(), req, url, orig_resp, d)
def findHash_done(self, hash, req, url, orig_resp, d):
"""Use the returned hash to lookup the file in the cache.
If the hash was not found, the workflow skips down to download from
the mirror (L{startDownload}), or checks the freshness of an old
response if there is one.
@type hash: L{Hash.HashObject}
@param hash: the hash object containing the expected hash for the file
"""
if hash.expected() is None:
log.msg('Hash for %s was not found' % url)
# Send the old response or get a new one
if orig_resp:
self.check_freshness(req, url, orig_resp, d)
else:
self.startDownload([], req, hash, url, d)
else:
log.msg('Found hash %s for %s' % (hash.hexexpected(), url))
# Lookup hash in cache
locations = self.db.lookupHash(hash.expected(), filesOnly=True)
self.getCachedFile(hash, req, url, d, locations)
def check_freshness(self, req, url, orig_resp, d):
"""Send a HEAD to the mirror to check if the response from the cache is still valid.
@type req: L{twisted.web2.http.Request}
@param req: the initial request sent to the HTTP server by apt
@param url: the URI of the actual mirror request
@type orig_resp: L{twisted.web2.http.Response}
@param orig_resp: the response from the cache to be sent to apt
"""
log.msg('Checking if %s is still fresh' % url)
modtime = orig_resp.headers.getHeader('Last-Modified')
headDefer = self.peers.get(HashObject(),
url,
method="HEAD",
modtime=modtime)
headDefer.addCallbacks(self.check_freshness_done,
self.check_freshness_error,
callbackArgs=(req, url, orig_resp, d),
errbackArgs=(req, url, d))
def check_freshness_done(self, resp, req, url, orig_resp, d):
"""Return the fresh response, if stale start to redownload.
@type resp: L{twisted.web2.http.Response}
@param resp: the response from the mirror to the HEAD request
@type req: L{twisted.web2.http.Request}
@param req: the initial request sent to the HTTP server by apt
@param url: the URI of the actual mirror request
@type orig_resp: L{twisted.web2.http.Response}
@param orig_resp: the response from the cache to be sent to apt
"""
if resp.code == 304:
log.msg('Still fresh, returning: %s' % url)
d.callback(orig_resp)
else:
log.msg('Stale, need to redownload: %s' % url)
self.startDownload([], req, HashObject(), url, d)
def check_freshness_error(self, err, req, url, d):
"""Mirror request failed, continue with download.
@param err: the response from the mirror to the HEAD request
@type req: L{twisted.web2.http.Request}
@param req: the initial request sent to the HTTP server by apt
@param url: the URI of the actual mirror request
"""
log.err(err)
self.startDownload([], req, HashObject(), url, d)
def getCachedFile(self, hash, req, url, d, locations):
"""Try to return the file from the cache, otherwise move on to a DHT lookup.
@type locations: C{list} of C{dictionary}
@param locations: the files in the cache that match the hash,
the dictionary contains a key 'path' whose value is a
L{twisted.python.filepath.FilePath} object for the file.
"""
if not locations:
log.msg('Failed to return file from cache: %s' % url)
self.lookupHash(req, hash, url, d)
return
# Get the first possible location from the list
file = locations.pop(0)['path']
log.msg('Returning cached file: %s' % file.path)
# Get it's response
resp = static.File(file.path).renderHTTP(req)
if isinstance(resp, defer.Deferred):
resp.addBoth(self._getCachedFile, hash, req, url, d, locations)
else:
self._getCachedFile(resp, hash, req, url, d, locations)
def _getCachedFile(self, resp, hash, req, url, d, locations):
"""Check the returned response to be sure it is valid."""
if isinstance(resp, failure.Failure):
log.msg('Got error trying to get cached file')
log.err(resp)
# Try the next possible location
self.getCachedFile(hash, req, url, d, locations)
return
log.msg('Cached response: %r' % resp)
if resp.code >= 200 and resp.code < 400:
d.callback(resp)
else:
# Try the next possible location
self.getCachedFile(hash, req, url, d, locations)
def lookupHash(self, req, hash, url, d):
"""Lookup the hash in the DHT."""
log.msg('Looking up hash in DHT for file: %s' % url)
key = hash.expected()
lookupDefer = self.dht.get(key)
lookupDefer.addBoth(self.startDownload, req, hash, url, d)
def startDownload(self, values, req, hash, url, d):
"""Start the download of the file.
The download will be from peers if the DHT lookup succeeded, or
from the mirror otherwise.
@type values: C{list} of C{dictionary}
@param values: the returned values from the DHT containing peer
download information
"""
# Remove some headers Apt sets in the request
req.headers.removeHeader('If-Modified-Since')
req.headers.removeHeader('Range')
req.headers.removeHeader('If-Range')
if not isinstance(values, list) or not values:
if not isinstance(values, list):
log.msg('DHT lookup for %s failed with error %r' %
(url, values))
else:
log.msg('Peers for %s were not found' % url)
getDefer = self.peers.get(hash, url)
# getDefer.addErrback(self.final_fallback, hash, url)
getDefer.addCallback(self.cache.save_file, hash, url)
getDefer.addErrback(self.cache.save_error, url)
getDefer.addCallbacks(d.callback, d.errback)
else:
log.msg('Found peers for %s: %r' % (url, values))
# Download from the found peers
getDefer = self.peers.get(hash, url, values)
getDefer.addCallback(self.check_response, hash, url)
getDefer.addCallback(self.cache.save_file, hash, url)
getDefer.addErrback(self.cache.save_error, url)
getDefer.addCallbacks(d.callback, d.errback)
def check_response(self, response, hash, url):
"""Check the response from peers, and download from the mirror if it is not."""
if response.code < 200 or response.code >= 300:
log.msg(
'Download from peers failed, going to direct download: %s' %
url)
getDefer = self.peers.get(hash, url)
# getDefer.addErrback(self.final_fallback, hash, url)
return getDefer
return response
def final_fallback(self, err, hash, url):
"""Final retry if the mirror still generated an error."""
log.msg('Download from mirror failed, retrying once only: %s' % url)
log.err(err)
getDefer = self.peers.get(hash, url)
return getDefer
def new_cached_file(self,
file_path,
hash,
new_hash,
url=None,
forceDHT=False):
"""Add a newly cached file to the mirror info and/or the DHT.
If the file was downloaded, set url to the path it was downloaded for.
Doesn't add a file to the DHT unless a hash was found for it
(but does add it anyway if forceDHT is True).
@type file_path: L{twisted.python.filepath.FilePath}
@param file_path: the location of the file in the local cache
@type hash: L{Hash.HashObject}
@param hash: the original (expected) hash object containing also the
hash of the downloaded file
@type new_hash: C{boolean}
@param new_hash: whether the has was new to this peer, and so should
be added to the DHT
@type url: C{string}
@param url: the URI of the location of the file in the mirror
(optional, defaults to not adding the file to the mirror info)
@type forceDHT: C{boolean}
@param forceDHT: whether to force addition of the file to the DHT
even if the hash was not found in a mirror
(optional, defaults to False)
"""
if url:
self.mirrors.updatedFile(url, file_path)
if self.my_addr and hash and new_hash and (hash.expected() is not None
or forceDHT):
return self.dht.store(hash)
return None
class senderAMQP:
def __init__(self, client, logger):
self.client = client # Client object (give access to all configuration options)
self.timeout = client.timeout # No timeout for now
self.logger = logger # Logger object
self.connection = None # The connection
self.igniter = None
self.ssl = False
self.reader = DiskReader(PXPaths.TXQ + self.client.name, self.client.batch,
self.client.validation, self.client.patternMatching,
self.client.mtime, True, self.logger, eval(self.client.sorter), self.client)
self.debugFile = False
self.cacheManager = CacheManager(maxEntries=self.client.cache_size, timeout=8*3600)
# AMQP is there a max for message size
# self.set_maxLength(self.client.maxLength)
# statistics.
self.totBytes = 0
self.initialTime = time.time()
self.finalTime = None
self._connect()
def printSpeed(self):
elapsedTime = time.time() - self.initialTime
speed = self.totBytes/elapsedTime
self.totBytes = 0
self.initialTime = time.time()
return "Speed = %i" % int(speed)
def setIgniter(self, igniter):
self.igniter = igniter
def resetReader(self):
self.reader = DiskReader(PXPaths.TXQ + self.client.name, self.client.batch,
self.client.validation, self.client.patternMatching,
self.client.mtime, True, self.logger, eval(self.client.sorter), self.client)
def _connect(self):
self.connection = None
self.channel = None
while True:
try:
host = self.client.host
if self.client.port != None : host = host + ':' + self.client.port
# connect
self.connection = amqp.Connection(host, userid=self.client.user, password=self.client.passwd, ssl=self.ssl)
self.channel = self.connection.channel()
# what kind of exchange
self.channel.access_request(self.client.exchange_realm, active=True, write=True)
self.channel.exchange_declare(self.client.exchange_name, self.client.exchange_type, auto_delete=False)
self.logger.info("AMQP Sender is now connected to: %s" % str(self.client.host))
break
except:
(type, value, tb) = sys.exc_info()
self.logger.error("AMQP Sender cannot connected to: %s" % str(self.client.host))
self.logger.error("Type: %s, Value: %s, Sleeping 5 seconds ..." % (type, value))
time.sleep(5)
def shutdown(self):
pass
def read(self):
if self.igniter.reloadMode == True:
# We assign the defaults and reread the configuration file (in __init__)
if self.channel != None : self.channel.close()
if self.connection != None : self.connection.close()
self.client.__init__(self.client.name, self.client.logger)
self.resetReader()
self.cacheManager.clear()
self.logger.info("Cache has been cleared")
self.logger.info("Sender AMQP has been reloaded")
self.igniter.reloadMode = False
self.reader.read()
return self.reader.getFilesContent(self.client.batch)
def write(self, data):
if len(data) >= 1:
self.logger.info("%d new messages will be sent", len(data) )
for index in range(len(data)):
self.logger.start_timer()
# data info
msg_body = data[index]
nbBytesSent = len(msg_body)
# if in cache than it was already sent... nothing to do
# priority 0 is retransmission and is never suppressed
path = self.reader.sortedFiles[index]
priority = path.split('/')[-3]
if self.client.nodups and priority != '0' and self.in_cache( data[index], True, path ) :
#PS... same bug as in Senders AM, AMIS & WMO.
#self.unlink_file( self.reader.sortedFiles[index] )
continue
# get/check destination Name
basename = os.path.basename(path)
destName, destDir = self.client.getDestInfos(basename)
if not destName :
os.unlink(path)
self.logger.info('No destination name: %s has been erased' % path)
continue
# build message
parts = basename.split(':')
if parts[-1][0:2] == '20' : parts = parts[:-1]
hdr = {'filename': ':'.join(parts) }
msg = amqp.Message(msg_body, content_type= self.client.exchange_content,application_headers=hdr)
# exchange_key pattern
exchange_key = self.client.exchange_key
if '$' in self.client.exchange_key :
exchange_key = self.keyPattern(basename,self.client.exchange_key)
self.logger.debug("exchange key = %s" % exchange_key)
# publish message
self.channel.basic_publish(msg, self.client.exchange_name, exchange_key )
self.logger.delivered("(%i Bytes) Message %s delivered" % (nbBytesSent, basename),path,nbBytesSent)
self.unlink_file( path )
self.totBytes += nbBytesSent
else:
time.sleep(1)
def run(self):
while True:
data = self.read()
try:
self.write(data)
except:
(type, value, tb) = sys.exc_info()
self.logger.error("Sender error! Type: %s, Value: %s" % (type, value))
# We close the connection
try:
self.channel.close()
self.connection.close()
except:
(type, value, tb) = sys.exc_info()
self.logger.error("Problem in closing socket! Type: %s, Value: %s" % (type, value))
# We try to reconnect.
self._connect()
#time.sleep(0.2)
# check if data in cache... if not it is added automatically
def in_cache(self,data,unlink_it,path):
already_in = False
# If data is already in cache, we don't send it
if self.cacheManager.find(data, 'md5') is not None:
already_in = True
if unlink_it :
try:
os.unlink(path)
self.logger.info("suppressed duplicate send %s", os.path.basename(path))
except OSError, e:
(type, value, tb) = sys.exc_info()
self.logger.info("in_cache unable to unlink %s ! Type: %s, Value: %s"
% (path, type, value))
return already_in
DirOSM = get_conf(configfile, "DirOSM") #~/greendrive/osmdata
DirData = replace_user_home(DirData)
DirOSM = replace_user_home(DirOSM)
gpsfolder = "gps"
obdfolder = "obd"
combinefolder = "combine"
MyIp = get_my_ip()
My_Platform = get_platform() # "centos" means cluster
On_Cluster = False
if My_Platform == 'centos': On_Cluster = True
iprint = 2
err = ErrorLogger("allerror.txt", py_fname(__file__, False))
lg = SimpleAppendLogger("./logs/" + py_fname(__file__, False))
mm = CacheManager() # using redis on cluster
EXT = get_conf(configfile, "EXT") # .gz
CUT = get_conf(configfile, "CUT") # ~|
EQU = get_conf(configfile, "EQU", delimiter=":")
KeyUserEmail = get_conf(configfile, "KeyUserEmail")
KeyUserName = get_conf(configfile, "KeyUserName")
UnknownUserEmail = get_conf(configfile, "UnknownUserEmail") # Anonymous
KeySysMs = get_conf(configfile, "KeySysMs")
KeyGPSTime = get_conf(configfile, "KeyGPSTime")
KeyGPSLat = get_conf(configfile, "KeyGPSLat")
KeyGPSLng = get_conf(configfile, "KeyGPSLng")
KeyGPSAccuracy = get_conf(configfile, "KeyGPSAccuracy")
KeyGPSSpeed = get_conf(configfile, "KeyGPSSpeed")
KeyGPSBearing = get_conf(configfile, "KeyGPSBearing")
KeyGPSAltitude = get_conf(configfile, "KeyGPSAltitude")
class WebSearcher:
def __init__(self):
self.cm = False
self.sleeptime = 2 #キャッシュ生成後にsleepする秒数
self.swWrited = False #検索結果のキャッシュを強制的に上書きするか(swWritedがTrueの場合に有効)
self.__params = {}
def activeCache(self, cachedir):
"""キャッシュ保存先の設定とキャッシュ保存関数の有効化"""
self.cm = CacheManager(cachedir)
def getHitCount(self):
"""get the number of retrieved pages
@return int the number of retrieved pages
"""
hits = False
if self.__engine == "tsubaki":
url = "http://tsubaki.ixnlp.nii.ac.jp/api.cgi?query=%s&only_hitcount=1" % self.__query
force_dpnd = self.getParameter("force_dpnd")
if force_dpnd:
url += "&force_dpnd=" + str(force_dpnd)
print url
if self.is_available_caching():
res = self.runCacheFunc(url)
else:
res = urllib.urlopen(url).read()
if not re.match("^[0-9]+$", res):
return "error"
hits = res.rstrip()
elif self.__engine == "yahoo" or self.__engine == "yahoo2":
if self.is_available_caching():
xmlstr = self.runCacheFunc(self.getResultURI(1, 1))
print xmlstr
doc = ElementTree(fromstring(xmlstr))
else:
url = self.getResultURI(1, 1)
fd = file(url, "rb")
doc = ElementTree(file=fd)
e = doc.getroot()
hits = e.attrib["totalResultsAvailable"]
return hits
def getResultURI(self, start, num):
engine = self.getEngine()
if engine == "tsubaki":
url = "http://tsubaki.ixnlp.nii.ac.jp/api.cgi?query=" + self.getQuery() + "&start=" + str(start) + "&results=" + str(num) + "&Snippet=1"
elif engine == "yahoo":
appid = self.getParameter("appid")
if not appid:
sys.exit("You must assign appid!!")
type = self.getParameter("type")
if not type:
type = "all"
url = "http://search.yahooapis.jp/WebSearchService/V1/webSearch?query=%s&appid=%s&type=%s&start=%s&results=%s&format=any" % (self.__query, appid, type, start, num)
print url
elif engine == "yahoo2":
appid = self.getParameter("appid")
if not appid:
sys.exit("You must assign appid!!")
type = self.getParameter("type")
if not type:
type = "all"
url = "http://search.yahooapis.jp/WebSearchService/V2/webSearch?query=%s&appid=%s&type=%s&start=%s&results=%s&format=html" % (self.__query, appid, type, start, num)
else:
url = False
return url
#"?query=" + urlencode(fnkf("-w", $this->query)) \
def is_available(self, engine):
"""check which defined search engine is available"""
available_engines = ["okwave", "tsubaki", "yahoo", "yahooQA"]
return engine in available_engines
def is_available_caching(self):
return self.cm != False
def runCacheFunc(self, url):
return self.cm.rwCache(url, self.sleeptime, self.swWrited)
def search(self, start, max):
"""run searching Web"""
engine = self.getEngine()
result = {}
rank = 1
if engine == "tsubaki":
url = self.getResultURI(start, max)
if self.is_available_caching():
xmlstr = self.runCacheFunc(url)
doc = ElementTree(fromstring(xmlstr))
else:
fd = file(url, "rb")
doc = ElementTree(file=fd)
for e in doc.findall("//Result"):
tmp = {"title":e.find("Title").text, "url":e.find("Url").text, "snippet":e.find("Snippet").text, "cache":"http://tsubaki.ixnlp.nii.ac.jp/api.cgi?id=" + e.attrib["Id"] + "&format=html"}
result[rank] = tmp
rank += 1
return result
def getEngine(self):
return self.__engine
def setEngine(self, engine):
self.__engine = engine
def getParameter(self, name):
if self.__params.has_key(name):
return self.__params[name]
else:
return False
def setParameter(self, name, value):
self.__params[name] = value
def getQuery(self):
return self.__query
def setQuery(self, query):
self.__query = query
## プロパティ ##
engine = property(getEngine, setEngine)
query = property(getQuery, setQuery)
def fix_no_speed(addr, mm_loaded=False):
print("fix_no_speed():", addr, 'mm_loaded', mm_loaded)
bugNoSpeedFn = mypydir + "/cache/nospeed-%s.txt" % addr
if On_Cluster:
mm_tmp_nid2spd = Mem({
"num": 30,
"prefix": "~fx_n2spd~",
"expire": 86400 * 30
})
else:
mm_tmp_nid2spd = Mem({
"use_ips": ['localhost'],
"prefix": "~fx_n2spd~",
"expire": 86400 * 30
})
aleady_mm_loaded = mm_loaded
correctSpdFn = DirOSM + os.sep + addr + "/%s-nids-to-speed.txt" % addr
if not aleady_mm_loaded:
cnt = 0
pthresh = 1
print('Loading ' + correctSpdFn)
with open(correctSpdFn, "r") as f:
for l in f:
st = l.split(",")
if len(st) < 3: continue
mm_tmp_nid2spd.set((int(st[0]), int(st[1])), float(st[2]))
cnt += 1
if cnt >= pthresh:
print('fix_no_speed load cnt', cnt)
pthresh *= 2
print("correct spd tup len=", cnt)
mm_nid2latlng = CacheManager(
overwrite_prefix=True) # will use existing config
mm_nid2neighbor = CacheManager(overwrite_prefix=True)
mm_nid2latlng.use_cache(
meta_file_name="osm/cache-%s-nodeid-to-lat-lng.txt" % addr)
mm_nid2neighbor.use_cache(
meta_file_name="osm/cache-%s-nodeid-to-neighbor-nid.txt" % addr)
n1n2s = {}
print('Reading ' + bugNoSpeedFn)
with open(bugNoSpeedFn, "r") as f:
for l in f:
st = l.split(",")
if len(st) < 2: continue
st = [int(x) for x in st]
tup = (st[0], st[1])
if tup not in n1n2s:
n1n2s[tup] = -1
print("no_speed nid tup len=", len(n1n2s))
lastBugCnt = None
while True:
bugcnt = 0
for tup in n1n2s.keys():
if n1n2s[tup] > 0:
continue
n1 = tup[0]
n2 = tup[1]
hd1 = get_bear_given_nid12(n1, n2, mm_nid2latlng)
if hd1 is None: continue
nblist = mm_nid2neighbor.get(n1) # ?->n1->n2
mindiff = 1e10
fixed = 0
if nblist:
for nbn in nblist:
hdn = get_bear_given_nid12(nbn, n1, mm_nid2latlng)
if hdn is None: continue
angle = min_angle_diff(hd1, hdn)
if (nbn, n1) in n1n2s and n1n2s[(nbn, n1)] > 0:
spdn = n1n2s[(nbn, n1)]
else:
spdn = mm_tmp_nid2spd.get((nbn, n1))
if angle < mindiff and spdn is not None:
mindiff = angle
n1n2s[tup] = spdn
fixed = 1
if fixed: continue
nblist = mm_nid2neighbor.get(n2) # n1->n2->?
mindiff = 1e10
if nblist:
for nbn in nblist:
hdn = get_bear_given_nid12(n2, nbn, mm_nid2latlng)
if hdn is None: continue
angle = min_angle_diff(hd1, hdn)
if (n2, nbn) in n1n2s and n1n2s[(n2, nbn)] > 0:
spdn = n1n2s[(n2, nbn)]
else:
spdn = mm_tmp_nid2spd.get((n2, nbn))
if angle < mindiff and spdn is not None:
mindiff = angle
n1n2s[tup] = spdn
fixed = 1
if fixed == 0: bugcnt += 1
if bugcnt == 0:
break
print("bugcnt", bugcnt)
if lastBugCnt is not None:
if lastBugCnt == bugcnt:
print("Give up #", bugcnt)
break
lastBugCnt = bugcnt
with open(correctSpdFn, "a") as f:
for tup in n1n2s.keys():
if n1n2s[tup] < 0: continue
print("%d,%d,%.2f" % (tup[0], tup[1], n1n2s[tup]))
f.write("%d,%d,%.2f\n" % (tup[0], tup[1], n1n2s[tup]))
print("Give up #", bugcnt)
class senderWmo(gateway.gateway):
def __init__(self,path,client,logger):
gateway.gateway.__init__(self, path, client, logger)
self.client = client
self.establishConnection()
self.reader = DiskReader(PXPaths.TXQ + self.client.name,
self.client.batch, # Number of files we read each time
self.client.validation, # name validation
self.client.patternMatching, # pattern matching
self.client.mtime, # we don't check modification time
True, # priority tree
self.logger,
eval(self.client.sorter),
self.client)
# Mechanism to eliminate multiple copies of a bulletin
self.cacheManager = CacheManager(maxEntries=self.client.cache_size, timeout=8*3600)
# WMO's maximum bulletin size is 500 000 bytes
self.set_maxLength( self.client.maxLength )
def set_maxLength(self,value):
if value <= 0 : value = 500000
self.maxLength = value
def shutdown(self):
gateway.gateway.shutdown(self)
resteDuBuffer, nbBullEnv = self.unSocketManagerWmo.closeProperly()
self.write(resteDuBuffer)
self.logger.info("Le senderWmo est mort. Traitement en cours reussi.")
def establishConnection(self):
# Instanciation du socketManagerWmo
self.logger.debug("Instanciation du socketManagerWmo")
self.unSocketManagerWmo = \
socketManagerWmo.socketManagerWmo(
self.logger,type='master', \
port=self.client.port,\
remoteHost=self.client.host,
timeout=self.client.timeout,
flow=self.client)
def read(self):
if self.igniter.reloadMode == True:
# We assign the defaults and reread the configuration file (in __init__)
self.client.__init__(self.client.name, self.client.logger)
self.set_maxLength( self.client.maxLength )
self.resetReader()
self.cacheManager.clear()
self.logger.info("Cache has been cleared")
self.logger.info("Sender WMO has been reloaded")
self.igniter.reloadMode = False
self.reader.read()
return self.reader.getFilesContent(self.client.batch)
def write(self,data):
#self.logger.info("%d nouveaux bulletins sont envoyes",len(data))
self.logger.info("%d new bulletins will be sent", len(data))
for index in range(len(data)):
self.logger.start_timer()
path = self.reader.sortedFiles[index]
basename = os.path.basename( path )
try:
tosplit = self.need_split( data[index] )
# need to be segmented...
if tosplit :
succes, nbBytesSent = self.write_segmented_data( data[index], path )
# all parts were cached... nothing to do
if succes and nbBytesSent == 0 :
self.logger.delivered("(%i Bytes) Bulletin %s delivered" % (len(data[index]), basename),path)
self.unlink_file( path )
continue
# send the entire bulletin
else :
# if in cache than it was already sent... nothing to do
# priority 0 are retransmission and no check for duplicate
priority = path.split('/')[-3]
if self.client.nodups and priority != '0' and self.in_cache( data[index], True, path ) :
#PS... same extra unlink as in AM sender call above is true, should it be false?
#self.unlink_file( self.reader.sortedFiles[index] )
continue
succes, nbBytesSent = self.write_data( data[index] )
#If the bulletin was sent successfully, erase the file.
if succes:
self.logger.delivered("(%i Bytes) Bulletin %s delivered" % (nbBytesSent, basename),path,nbBytesSent)
self.unlink_file( path )
else:
self.logger.info("%s: Sending problem" % path )
except Exception, e:
# e==104 or e==110 or e==32 or e==107 => connection broken
(type, value, tb) = sys.exc_info()
self.logger.error("Type: %s, Value: %s" % (type, value))
# Log infos about tx speed
if (self.totBytes > 1000000):
self.logger.info(self.printSpeed() + " Bytes/sec")
# Log infos about caching
(stats, cached, total) = self.cacheManager.getStats()
if total:
percentage = "%2.2f %% of the last %i requests were cached (implied %i files were deleted)" % (cached/total * 100, total, cached)
else:
percentage = "No entries in the cache"
self.logger.info("Caching stats: %s => %s" % (str(stats), percentage))
def ingestBulletinFile(self, igniter):
from DiskReader import DiskReader
import bulletinManager
import bulletinManagerAm
from PullFTP import PullFTP
sleep_sec = 1
if self.source.type == 'pull-bulletin' or self.source.pull_script != None : sleep_sec = self.source.pull_sleep
bullManager = bulletinManager.bulletinManager(
self.ingestDir,
self.logger,
self.ingestDir,
99999,
'\n',
self.source.extension,
self.source.routingTable,
self.source.mapEnteteDelai,
self.source,
self.source.addStationInFilename)
if self.source.bulletin_type == 'am' :
bullManager = bulletinManagerAm.bulletinManagerAm(
self.ingestDir,
self.logger,
self.ingestDir,
99999,
'\n',
self.source.extension,
self.source.routingTable,
self.source.addSMHeader,
PXPaths.STATION_TABLE,
self.source.mapEnteteDelai,
self.source,
self.source.addStationInFilename)
if self.source.nodups :
self.fileCache = CacheManager(maxEntries=self.source.cache_size, timeout=8*3600)
reader = DiskReader(bullManager.pathSource, self.source.batch, self.source.validation, self.source.patternMatching,
self.source.mtime, False, self.source.logger, self.source.sorter, self.source)
while True:
# If a SIGHUP signal is received ...
if igniter.reloadMode == True:
# We assign the defaults, reread configuration file for the source
# and reread all configuration file for the clients (all this in __init__)
if self.source.type == 'filter-bulletin' :
self.source.__init__(self.source.name, self.source.logger, True, True)
else :
self.source.__init__(self.source.name, self.source.logger)
bullManager = bulletinManager.bulletinManager(
self.ingestDir,
self.logger,
self.ingestDir,
99999,
'\n',
self.source.extension,
self.source.routingTable,
self.source.mapEnteteDelai,
self.source,
self.source.addStationInFilename)
if self.source.bulletin_type == 'am' :
bullManager = bulletinManagerAm.bulletinManagerAm(
self.ingestDir,
self.logger,
self.ingestDir,
99999,
'\n',
self.source.extension,
self.source.routingTable,
self.source.addSMHeader,
PXPaths.STATION_TABLE,
self.source.mapEnteteDelai,
self.source,
self.source.addStationInFilename)
if self.source.nodups :
self.fileCache = CacheManager(maxEntries=self.source.cache_size, timeout=8*3600)
reader = DiskReader(bullManager.pathSource, self.source.batch, self.source.validation, self.source.patternMatching,
self.source.mtime, False, self.source.logger, self.source.sorter,self.source)
self.logger.info("Receiver has been reloaded")
igniter.reloadMode = False
# pull files in rxq directory if in pull mode
if self.source.type == 'pull-bulletin' or self.source.pull_script != None :
files = []
sleeping = os.path.isfile(PXPaths.RXQ + self.source.name + '/.sleep')
if self.source.type == 'pull-bulletin' :
puller = PullFTP(self.source,self.logger,sleeping)
files = puller.get()
puller.close()
elif self.source.pull_script != None :
files = self.source.pull_script(self.source,self.logger,sleeping)
if not sleeping :
self.logger.debug("Number of files pulled = %s" % len(files) )
else :
self.logger.info("This pull is sleeping")
# normal diskreader call for files
reader.read()
# processing the list if necessary...
if self.source.lx_execfile != None and len(reader.sortedFiles) > 0:
sfiles = []
sfiles.extend(reader.sortedFiles)
self.logger.info("%d files process with lx_script" % len(sfiles))
sortedFiles = self.source.run_lx_script(sfiles,self.source.logger)
reader.sortedFiles = sortedFiles
# continue normally
data = reader.getFilesContent(reader.batch)
if len(data) == 0:
time.sleep(sleep_sec)
continue
else:
self.logger.info("%d bulletins will be ingested", len(data))
# Write (and name correctly) the bulletins to disk, erase them after
for index in range(len(data)):
# ignore duplicate if requiered
duplicate = self.source.nodups and self.fileCache.find(data[index], 'md5') is not None
#nb_bytes = len(data[index])
#self.logger.info("Lecture de %s: %d bytes" % (reader.sortedFiles[index], nb_bytes))
if not duplicate :
# converting the file if necessary
if self.source.fx_execfile != None :
file = reader.sortedFiles[index]
fxfile = self.source.run_fx_script(file,self.source.logger)
# convertion did not work
if fxfile == None :
self.logger.warning("FX script ignored the file : %s" % os.path.basename(file) )
os.unlink(file)
continue
# file already in proper format
elif fxfile == file :
self.logger.warning("FX script kept the file as is : %s" % os.path.basename(file) )
# file converted...
else :
self.logger.info("FX script modified %s to %s " % (os.path.basename(file),os.path.basename(fxfile)) )
os.unlink(file)
fp = open(fxfile,'r')
dx = fp.read()
fp.close()
reader.sortedFiles[index] = fxfile
data[index] = dx
# writing/ingesting the bulletin
if isinstance(bullManager,bulletinManagerAm.bulletinManagerAm):
bullManager.writeBulletinToDisk(data[index], True)
else :
bullManager.writeBulletinToDisk(data[index], True, True)
try:
file = reader.sortedFiles[index]
os.unlink(file)
if duplicate : self.logger.info("suppressed duplicate file %s", os.path.basename(file))
self.logger.debug("%s has been erased", os.path.basename(file))
except OSError, e:
(type, value, tb) = sys.exc_info()
self.logger.error("Unable to unlink %s ! Type: %s, Value: %s" % (reader.sortedFiles[index], type, value))
class unittest_CacheManager(unittest.TestCase):
def setUp(self):
self.manager = CacheManager(maxEntries=3, timeout=5 * 3600)
self.files = ['db/test-file-grib', 'db/test-file-burf', 'db/test-file-bulletin']
#f = open(self.files[0])
#self.data = f.read(1048576)
#while len(self.data) :
#print self.data
#self.data = f.read(1048576)
#f.close()
self.data = 'ceci est un test'
def test_CacheManager(self):
self.assertEqual(self.manager.get_md5_from_file(self.files[0]),'57285445a1c80023b3f2e96546754d5b')
self.manager.find(self.data,'md5')
self.manager.find(self.files[1])
#md5 of self.data = 11b35a0201513381dcdd130831f702d0
self.assertEqual(self.manager.has('11b35a0201513381dcdd130831f702d0'),True)
self.assertEqual(self.manager.has(self.files[2]),False)
self.manager.find(self.data,'md5')
self.manager.find(self.files[1])
self.manager.find(self.files[2])
self.assertEqual(self.manager.getStats(),({1: 1, 2: 2}, 2.0, 5.0))
self.manager.find(self.files[0])
time.sleep(3)
self.manager.timeoutClear(3)
self.assertEqual(self.manager.cache,{})
'AdmBldg22AP1': 19, 'AcadBldg30AP3': 1, 'ResBldg82AP3': 3, 'AcadBldg8AP4': 14,
'LibBldg2AP19': 8, 'AcadBldg30AP1': 10, 'LibBldg2AP17': 13, 'LibBldg1AP17': 2,
'LibBldg2AP13': 2, 'LibBldg2AP10': 3, 'LibBldg2AP11': 3}
}
userMovement = {
24: {'SocBldg2AP1': 10, 'SocBldg3AP2': 8, 'AcadBldg22AP2': 0, 'LibBldg4AP3': 4}
}
tp = TopologyGenerator('/home/ubuntu/Downloads/APlocations_clean.csv')
networkManager = NetworkManager()
apsByBuildings, buildingNames = tp.getSample()
linkage = {} #tp.computeLinkage(printDendogram = False)
clusters = {} #tp.computeClusters()
net, tree = networkManager.networkFromCLusters('test', clusters, linkage, len(buildingNames), apsByBuildings, buildingNames)
print '*** Getting requests data'
#movementParser = MovementDataParser('/home/ubuntu/Downloads/movement/2001-2003/', '/data/movement.csv')
#movementParser.getMovementInfo()
cacheManager = CacheManager(tree)
print cacheManager.computeKMedianCaches(k=2, userId=24, userMovement=userMovement)
#networkManager.simulation(net)
#CLI( net )
#net.stop()
