def __init__(self,
num_processes=2,
resource_lifespan=86400,
job_lifespan=86400):
self.allowed_extensions = {'csv'}
self.resource_lifespan = resource_lifespan
self.job_lifespan = job_lifespan
self.num_processes = num_processes
self.job_queue = None
self.consumers = None
self.manifest = Manifest()
def __init__(self, user, password, reader, writer, database):
self.manifest = Manifest()
self.builder = MysqlDB()
self.user = user
self.password = password
self.reader = reader
self.writer = writer
self.database = database
def __init__(self, requestQueue):
self.requestQueue = requestQueue
self.serverSocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.manifest = Manifest()
self.bufferSize = self.manifest.listener_buffer_size
self.portNumber = self.manifest.port_number
self.serverIp = ''
self.reqCount = 0
class ManifestTests(unittest.TestCase):
def setUp(self):
self.manifest = Manifest('static_manifest.xml')
f = open('static_manifest.xml')
data = f.read()
self.data = data
def test_content(self):
self.assertEqual(self.manifest.find_project(name='platform/build')\
['revision'], "81056a1c2bf8d42a9ca70eee90bb04e268dcddb1")
def test_save(self):
mf = 'test_manifest.xml'
if os.path.exists(mf):
os.remove(mf)
self.manifest.save(mf)
self.assertTrue(os.path.exists(mf))
def test_from_dict(self):
self.manifest.from_dict(self.manifest.M)
self.test_content()
self.test_save()
def test_from_string(self):
self.manifest.from_string(self.data)
self.test_content()
def test_constructor_with_data(self):
self.manifest = Manifest(self.data)
self.test_content
def parse_pdf(path, pages=134):
global LastBOL
try:
pdf_file = open(path, 'rb')
#Create PDF Parser
rsrcmanager = PDFResourceManager()
PDFPageAgg = PDFPageAggregator(rsrcmanager, laparams=LAParams())
interpreter = PDFPageInterpreter(rsrcmanager, PDFPageAgg)
for n, page in enumerate(
PDFPage.get_pages(pdf_file,
maxpages=pages,
password="",
caching=True)):
n += 1
print('###### page ' + str(n))
myTEUs = 0
myWeight = 0
interpreter.process_page(page)
# receive the LTPage object for the page.
LTPage_layout = PDFPageAgg.get_result()
myBOL = get_BOL_from_elements(LTPage_layout)
if not myBOL:
print("Empty Page " + "\n")
continue
if LastBOL == myBOL:
myTEUs, myWeight = get_TEUs_from_elements(LTPage_layout)
if myTEUs == 0:
print("Continue same BOL: " + myBOL + "\n")
continue
else:
myManifest.TEUs = myTEUs
myManifest.WeightORG_Tonne = myWeight
else:
myManifest = Manifest()
myManifest.BOL = myBOL
get_text_from_elements(LTPage_layout, myManifest)
if myManifest.TEUs > 0:
myManifest.SetCalculatedValues()
myManifest.myPrint()
Manifests.append(myManifest.get_list())
print("-------------------------------------")
else:
print("Continue same BOL: " + myBOL + "\n")
myTEUs = 0
LastBOL = myBOL
pdf_file.close()
PDFPageAgg.close()
except Exception:
traceback.print_exc()
def __init__(self, requestQueue, gameQueue, gameCollection):
self.manifest = Manifest()
reader = self.manifest.database_reader
writer = self.manifest.database_writer
username = self.manifest.database_username
password = self.manifest.database_password
gameDatabase = self.manifest.game_database_name
self.database = DB(username, password, reader, writer, gameDatabase)
#self.database = DB('app','123','192.168.1.106', '192.168.1.106','gamedb')
self.requestQueue = requestQueue
self.gameQueue = gameQueue
self.reqValidation = ValidateRequest()
self.responder = Responder()
self.gameCollection = gameCollection
self.gameCollection.setDatabase(self.database)
self.gameGenerator = GameGenerator(self.database, self.gameQueue,
self.gameCollection)
def test_constructor_with_data(self):
self.manifest = Manifest(self.data)
self.test_content
def setUp(self):
self.manifest = Manifest('static_manifest.xml')
f = open('static_manifest.xml')
data = f.read()
self.data = data
def main(argv = None):
if argv is None:
argv = sys.argv
try:
try:
options, arguments = getopt.getopt(argv[1:], "i:z:l:S:r:R:cm:d:v:e:")
caselist = Caselist()
manifest = Manifest()
keymap = Keymap()
# logging
log = logging.getLogger('process')
log.setLevel(logging.INFO)
log.addHandler(logging.StreamHandler())
# process options
for option, argument in options:
if option in ("-h", "--help"):
print __doc__
sys.exit(0)
if option in ("-i", "--input"):
if os.path.isfile(argument):
caselist.loadCasesFromPaths(manifest.read(argument))
elif os.path.isdir(argument):
caselist.loadCasesFromDir(argument)
else:
print "Error: case files not loaded."
sys.exit(0)
if option in ("-z", "--config"):
if os.path.isfile(argument):
Config.load(argument)
else:
print "Error: config file not loaded."
sys.exit(0)
#### load ####
if option in ("-l", "--load-keymap"):
keymap.load(argument)
#### search/replace ####
if option in ("-S", "--search-by-key-value"):
keyValuePair = argument.split(':')
key = keyValuePair[0]
value = keyValuePair[1]
caselist.searchKeys(key, value)
if option in ("-r", "--substitute-key"):
substituteKeys = argument.split(':')
caselist.substituteKey(substituteKeys[0], substituteKeys[1])
if option in ("-R", "--subsitute-keys-from-file"):
caselist.substituteKeysFromFile(argument)
#### dump ####
if option in ("-c", "--dump-cases"):
caselist.dumpCases()
if option in ("-m", "--dump-keys-by-keymap"):
caselist.dumpGroupedKeys(keymap, argument)
if option in ("-d", "--dump-stata"):
caselist.dumpStataFiles(argument)
if option in ("-v", "--dump-stata-commands"):
keymap.dumpStataCommands(argument, caselist)
if option in ("-e", "--dump-excel-data"):
keymap.dumpExcelData(argument, caselist)
#### etc ####
# caselist.explore()
except getopt.error, message:
raise Usage(message)
except Usage, error:
print >>sys.stderr, error.message
print >>sys.stderr, "for help use --help"
return 2
class ResourceManager:
def __init__(self,
num_processes=2,
resource_lifespan=86400,
job_lifespan=86400):
self.allowed_extensions = {'csv'}
self.resource_lifespan = resource_lifespan
self.job_lifespan = job_lifespan
self.num_processes = num_processes
self.job_queue = None
self.consumers = None
self.manifest = Manifest()
def start(self):
if not os.path.exists('resources/'):
os.mkdir('resources/')
if not os.path.exists('jobs/'):
os.mkdir('jobs/')
self.job_queue = JoinableQueue()
self.consumers = [
Consumer(self.job_queue, self.manifest)
for _ in range(self.num_processes)
]
for consumer in self.consumers:
consumer.start()
def shutdown(self):
for consumer in self.consumers:
consumer.terminate()
self._remove_all()
def get_job_status(self, job_id):
return self.manifest.get_job_status(job_id)
def add_job_to_queue(self, job):
self.manifest.add_job(job.job_id)
self.job_queue.put(job)
def delete_job_results(self, job_id):
self.manifest.delete_job(job_id)
path = 'jobs/' + job_id
if os.path.exists(path):
try:
shutil.rmtree(path)
return True
except BaseException:
return False
return False
def delete_expired_resources(self):
expired_resources = self.manifest.delete_expired_resources(
self.resource_lifespan)
for resource_id in expired_resources:
try:
shutil.rmtree('resources/' + resource_id)
except:
pass
def delete_expired_jobs(self):
expired_jobs = self.manifest.delete_expired_jobs(self.job_lifespan)
for job_id in expired_jobs:
try:
shutil.rmtree('jobs/' + job_id)
except:
pass
@staticmethod
def get_zip_filename(job_id):
folder = 'jobs/' + job_id + '/'
if not os.path.exists(folder):
return None
zip_filename = folder + job_id + '.zip'
zip = zipfile.ZipFile(zip_filename, 'w')
for file in os.listdir(folder):
if not file.endswith('.zip'):
zip.write(folder + file)
return zip_filename
@staticmethod
def get_aggregated_data(job_id):
filename = 'jobs/' + job_id + '/aggregate.json'
if not os.path.exists(filename):
return None
with open(filename) as file:
return json.load(file)
@staticmethod
def job_id_is_safe(job_id):
return '/' not in job_id and '.' not in job_id
@staticmethod
def get_new_resource_id():
return uuid.uuid4().hex
@staticmethod
def get_resource_hash(resource_id):
filename = 'resources/' + resource_id
if not os.path.exists(filename):
return None
block_size = 1024
m = hashlib.sha256()
with open(filename, "rb") as file:
byte = file.read(block_size)
while byte != b"":
m.update(byte)
byte = file.read(block_size)
return m.hexdigest()
def add_resource(self, resource_id):
resource_hash = self.get_resource_hash(resource_id)
self.manifest.add_resource(resource_id, resource_hash)
def delete_resource(self, resource_id):
self.manifest.delete_resource(resource_id)
filepath = 'resources/' + resource_id
if not os.path.exists(filepath):
return False
os.remove(filepath)
return True
def extension_is_allowed(self, filename):
if '.' not in filename:
return False
extension = filename.split('.')[1]
return extension in self.allowed_extensions
def resource_id_exists(self, resource_id):
return self.manifest.resource_exists(resource_id)
def resource_hash_exists(self, resource_hash):
return self.manifest.resource_hash_exists(resource_hash)
def _remove_all(self):
self.manifest.clear()
shutil.rmtree('resources/')
shutil.rmtree('jobs/')
if os.path.exists('data/'):
if os.path.exists('data/osm_query_cache'):
shutil.rmtree('data/osm_query_cache')
