def _create_files(self):
"""Generate a new File object for every file the metainfo
file told us about.
"""
try:
yield (File(
self.piece_length,
self.info_dict['info']['name'],
self.info_dict['info']['length']))
except KeyError:
for f in self.info_dict['info']['files']:
yield (File(self.piece_length, f['path'], f['length']))
def _create_files(self):
"""Generate a new File object for every file the metainfo
file told us about.
"""
try:
yield (File(self.piece_length, self.info_dict['info']['name'],
self.info_dict['info']['length']))
self.logger.info('Appended file {} of length {}'.format(
self.info_dict['info']['name'],
self.info_dict['info']['length']))
except KeyError:
for f in self.info_dict['info']['files']:
self.logger.info('Appending file {} of length {}'.format(
f['path'][len(f['path']) - 1], f['length']))
yield (File(self.piece_length, f['path'], f['length']))
def StoreListings( listingObject ):
#Prepare the secondary dictionaries
listing_sales = listingObject.get( 'saleDetails' )
lisitng_inspections = listingObject.get( 'inspectionDetails' )
listing_prices = listingObject.get( 'priceDetails' )
try:
#Insert the raw listing first.
#Get the listing_status
listing_status = QueryWithSingleValue( 'listing_status_lkp', 'description', listingObject['status'], 'listing_status_id', True )
#build the JSON from listingObject
raw_listing_JSON = json.dumps( listingObject )
#Get the value that will be used with listing_insert_statement
listings_id = returnNextSerialID( 'listings', 'listings_id' )
if lisitng_inspections is not None:
isByAppointmentOnly = lisitng_inspections.get( 'isByAppointmentOnly' )
else:
isByAppointmentOnly = None
cur.execute( """ INSERT INTO listings( domain_listings_id, headline, price_displayed, display_price, price, price_from, price_to, seo_url, listing_objective,
listing_status, land_area, building_area, energy_efficiency, is_new_development, date_updated, date_created,
entered_when, entered_by, raw_listing, inspection_appointment_only )
VALUES( %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, to_timestamp( %s, 'YYYY-MM-DD HH24:MI:SS' ), to_timestamp( %s, 'YYYY-MM-DD HH24:MI:SS' ), current_timestamp, 1, %s, %s ) """,
( listingObject.get( 'id' ), listingObject.get( 'headline' ), listing_prices.get( 'canDisplayPrice' ), listing_prices.get( 'displayPrice' ), listing_prices.get( 'price' ),
listing_prices.get( 'priceFrom' ), listing_prices.get( 'priceTo' ), listingObject.get( 'seoUrl' ), listingObject.get( 'objective' ), listing_status, listingObject.get( 'landAreaSqm'),
listingObject.get( 'buildingAreaSqm' ), listingObject.get( 'energyEfficiencyRating' ), listingObject.get( 'isNewDevelopment' ), convertJSONDate(listingObject['dateUpdated']),
convertJSONDate(listingObject['dateListed']), cleanForSQL(raw_listing_JSON), isByAppointmentOnly ) )
#Insert the Features if the listing contains any.
#Set the object type
#Only do this if the listing already has a features object.
if 'features' in listingObject:
link_object_type = OBJECT_Listing
for feature in listingObject['features']:
storeFeatures( listings_id, link_object_type, feature )
if 'media' in listingObject:
#Store any media attached to the listing
for media in listingObject['media']:
mediaObject = File( FILE_TYPE_Images, OBJECT_Listing, str(listings_id), None, "listing_" + media['type'] )
mediaObject.addImageDetails( None, None, media['url'] )
mediaObject.storeFile( False )
#Store Listing Sales Information.
#First, we need to check if the listings has any sales information attached to it.
if listing_sales is not None:
storeListingSalesDetails( listing_sales, listings_id )
#Store the Inspection information (if the listing_inspections array is not None)
if lisitng_inspections is not None:
storeInspectionDetails( listings_id, lisitng_inspections )
return listings_id
except(Exception, psycopg2.DatabaseError) as error:
print("Error in INSERTING New Listing with Domain Listing ID " + "\n" + "Error: " + error )
return None
def __init__(self):
self.file = File()
self.file.file_menu()
self.run = tk.Menu(menubar.toolbar, tearoff=False)
self.run.add_command(label="Run", command=lambda: self.run_it())
menubar.toolbar.add_cascade(label="Run", menu=self.run)
self.binding_keys()
def create_file():
global makeChanges
file_name = input("Enter File name: ")
file = File(file_id_assigner(), file_name, 0, {})
JSON_structure["files"].update(file.create_f())
print("File Created Successfully!")
makeChanges = 1
def main():
os.chdir(path)
html = HTML(url="http://www.genenames.org/cgi-bin/hgnc_downloads.cgi"
) # Check html for attributes.
attributes = html.find_between(
"</td> <td>", "</td>", '"',
all=True) # Retrieve all aviable attributes.
print("Number of attributes: %s" %
len(attributes)) # Check number of attributes.
# Building url:
url_begin = "http://www.genenames.org/cgi-bin/hgnc_downloads.cgi?title=Core+Data"
url_context = ";col=" + ";col=".join(
attributes
) #col=gd_hgnc_id;col=gd_app_sym;col=gd_app_name;col=gd_status;col=gd_prev_sym;col=gd_aliases;col=gd_pub_chrom_map;col=gd_pub_acc_ids;col=gd_pub_refseq_ids;
url_end = ";status=Approved;status=Approved+Non-Human;status=Entry+Withdrawn;status_opt=3;=on;where=;order_by=gd_app_sym_sort;limit=;format=text;submit=submit;.cgifields=;.cgifields=status;.cgifields=chr"
url = url_begin + url_context + url_end
f = File(name="hgnc.txt", url=url, path=path)
contents = f.parse(printing=False, header=True)
genes.name = "HGNC"
genes.key = "hgnc"
genes.taxid = 9606
genes.addData(contents)
genes.save()
genes.buildMappings()
def initFromObject(cls, propertyObject, propertyID=None):
if 'addressCoordinate' in propertyObject:
longitude = propertyObject['addressCoordinate']['lon']
lattitude = propertyObject['addressCoordinate']['lat']
else:
longitude = None
lattitude = None
photos = []
#Create File Array from the Photos Object inside the propertyObject
for photo in propertyObject['photos']:
propertyImage = File(FILE_TYPE_Images, OBJECT_Property, propertyID,
None, 'Property ' + photo['imageType'], None)
propertyImage.addImageDetails(photo['advertId'], photo['date'],
photo['fullUrl'], None)
photos.append(propertyImage)
new_property = cls(
propertyObject['id'], propertyObject.get("cadastreType"),
VarIf(propertyObject['status'] == PROPERTY_MARKET_STATUS_ONMARKET,
True, False),
Address(propertyObject['address'], propertyObject['streetName'],
propertyObject['suburb'], propertyObject['streetNumber'],
propertyObject.get("zone"),
propertyObject.get("lotNumber"), longitude, lattitude,
None), propertyObject.get("areaSize"),
propertyObject.get('numBedrooms'),
propertyObject.get('numBathrooms'), propertyObject.get('features'),
json.dumps(propertyObject), propertyObject.get('history'), photos)
return new_property
def convertToClasses(self):
"""
This Function converts all instance variables of the Agency Object that are meant to be classes...(i.e agency_banner) into their respective class.
(These includes the Contact Details and Files Classes)
"""
#Check to see if the instance has its agency_id set or has already had its variables converted into classes.
try:
if self.agency_id is None or self.classesConverted == True:
raise Exception(
"Cannot convert classes due to them already being converted or Agency Instance having a Null Agency ID"
)
else:
agency_banner = File(FILE_TYPE_Images, OBJECT_Agency,
self.agency_id, None, "agency_banner")
agency_banner.addImageDetails(None, None, self.agency_banner)
self.agency_banner = agency_banner
self.agency_website = ContactDetails('agency_website',
OBJECT_Agency,
self.agency_id,
self.agency_website)
agency_logo_standard = File(FILE_TYPE_Images, OBJECT_Agency,
self.agency_id, None,
"agency_logo_standard")
agency_logo_standard.addImageDetails(None, None,
self.agency_logo_standard)
self.agency_logo_standard = agency_logo_standard
self.contact_details_converted = []
for contactType, detailsType in self.contact_details.items():
#For now, only store dictionaries. We can handle exceptions by logging them to a debug file.
if isinstance(detailsType, dict):
for contact, details in detailsType.items():
if details != '':
self.contact_details_converted.append(
ContactDetails(
'agency_' + contactType + '_' +
contact, OBJECT_Agency, self.agency_id,
details))
#else:
# self.contact_details_converted.append( ContactDetails( 'agency_' + contactType + '_' + contact, self.agency_id, OBJECT_Agency, details ) )
self.classesConverted = True
except (Exception) as error:
print(error)
def template(self):
"""
File's template. Note that this template is not associated to any
collection.
@rtype: L{File}
"""
return self._get_field("template", lambda x: File(None, x))
def decode(socket):
fixed = bytearray(6)
socket.recv_into(fixed, flags=MSG_WAITALL)
file_path_length = byte_utils.bytes_to_char(fixed, 0)
file_is_directory = byte_utils.bytes_to_boolean(fixed, 1)
file_last_modified = byte_utils.bytes_to_unsigned_int(fixed, 2)
file_size = None
if not file_is_directory:
fixed = bytearray(4)
socket.recv_into(fixed, flags=MSG_WAITALL)
file_size = byte_utils.bytes_to_unsigned_int(fixed, 0)
strings = bytearray(file_path_length)
socket.recv_into(strings, flags=MSG_WAITALL)
file_path = byte_utils.bytes_to_string(strings, file_path_length, 0)
if utils.DEBUG_LEVEL >= 3:
utils.log_message("DEBUG", "Decoded send file packet: ")
utils.log_message("DEBUG",
"File path length: " + str(file_path_length))
utils.log_message("DEBUG",
"Is directory: " + str(file_is_directory))
utils.log_message(
"DEBUG", "Last modified: " +
str(utils.format_timestamp(file_last_modified)))
utils.log_message("DEBUG", "File size: " + str(file_size))
utils.log_message("DEBUG", "File Path: " + str(file_path))
# parse file's contents to File().write() 1024 chunks if is not directory
if not file_is_directory:
chunk_size = min(SendFilePacket.CHUNK_SIZE, file_size)
remaining = file_size
file_wrapper = File()
received_bytes_acc = 0
while remaining > 0:
if utils.DEBUG_LEVEL >= 3:
utils.log_message("DEBUG",
"Chunk size: " + str(chunk_size))
chunk = bytearray(chunk_size)
received_bytes = socket.recv_into(chunk, flags=MSG_WAITALL)
received_bytes_acc += received_bytes
file_wrapper.write(chunk)
remaining -= received_bytes
chunk_size = min(chunk_size, remaining)
file_wrapper.close()
if utils.DEBUG_LEVEL >= 1:
utils.log_message(
"DEBUG", "File size is " + str(file_size) +
" and received bytes are " + str(received_bytes_acc))
utils.log_message(
"DEBUG",
"File is located in " + str(file_wrapper.get_path()))
else:
file_wrapper = Directory()
packet = SendFilePacket(
FileInfo(file_path, file_is_directory, file_last_modified,
file_size, file_wrapper))
return packet
def create_file(file_name):
global makeChanges
flag = False
for fileIndexes in list(JSON_structure["files"]):
if JSON_structure["files"][fileIndexes]["name"] == file_name:
flag = True
break
if flag:
display_msg("File with the same name already Exists!")
else:
file = File(file_id_assigner(), file_name, 0, {})
JSON_structure["files"].update(file.create_f())
JSON_structure["meta_data"]["files"] += 1
display_msg("File Created Successfully!")
makeChanges = 1
def files_():
logdir = os.path.join(os.getcwd(), "logs")
f = File("av_log.bin", path=logdir)
f.put({
"interested": [
"coet", "lluis_tgn", "Mafoso-Espieta", "krls-ca", "Vriullop",
"Lohen", "Paucabot", "SMP_ca"
],
"authorized": ["pasqual", "krls-ca", "Lohen", "SMP", "Vriullop"],
"usersQueue": [],
"channels": {}
})
f.close()
data = f.get()
print data
f.put(data)
def main():
"""Testing suits here:"""
#print Map.dbs
#print Map.it('zbtb16', 9606)
#print m("ZBTB16")
## res = m("eat-1", 6239)
## print res
## return res
print m('chico', 7227)
f = File(name='MappedIDs.txt')
data = f.parse(header=True, printing=False)
for i in data:
if not i: continue # correct this.
#print i
id = i['Input Id']
print id, m([id], 6239)[0],
try:
print mo([id], 6239) #[0]
except:
print
#print data[1]
print m("cha-1", 6239)
print m("unc-17", 6239)
def yield_dicts_from_file(path):
doc = File(path).read_text()
rows = doc_to_lists(doc)
return yield_rows_as_dicts(rows)
def __init__(self, path: str):
"""Read data from *path* file"""
doc = File(path).read_text()
self.rows = doc_to_lists(doc)
from files import File
from position_verify import put_position
from os import system
from move_verify import verify_move
from tqdm import tqdm
from move_files import Move
# here are created the objects
system("cls")
name_of_player_one = input("PLAYER ONE ➡️ your files are white" + "\n" +
"enter your name: ")
name_of_player_two = input("PLAYER TWO ➡️ your files are black" + "\n" +
"enter your name: ")
board_player = Board()
file_one = File("●", "◍")
file_two = File("○", "◌")
player_one_add = Player(board_player, file_one, name_of_player_one)
player_two_add = Player(board_player, file_two, name_of_player_two)
player_move_one = Move(board_player, file_one, name_of_player_one)
player_move_two = Move(board_player, file_two, name_of_player_two)
# in this function the momevents are ejecuted with the verification
quantl_files_1 = 0
quantl_files_2 = 0
def movements(board_2, names):
# this variables are for control the quantity of the files. if the player has three files
global quantl_files_1, quantl_files_2
def storeAgent(self, commit):
try:
self.agent_id = returnNextSerialID('agents', 'agent_id')
#Stores information contained inside the Agent Object into the Agents table.
cur.execute(
"""INSERT INTO agents( domain_agent_id, entered_when, first_name, last_name, profile_text )
VALUES( %s, current_timestamp, %s, %s, %s )""",
(self.domain_agent_id, cleanForSQL(self.first_name),
cleanForSQL(self.last_name), self.profile_text))
#Store the link between the Agent and the Agency inside the agencies_agent table.
cur.execute(
"""INSERT INTO agencies_agent( agency_id, agent_id, entered_when )
VALUES( %s, %s, current_timestamp)""",
(self.agency_id, self.agent_id))
if self.email is not None:
#Store the agent's emaia
contactDetails_email = ContactDetails("agent_email",
OBJECT_Agent,
self.agent_id,
self.email)
if not contactDetails_email.storeContactDetails(False):
raise Exception(psycopg2.DatabaseError)
if self.phone is not None:
#Store the agent's phone number
contactDetails_phone = ContactDetails("agent_phone_number",
OBJECT_Agent,
self.agent_id,
self.phone)
if not contactDetails_phone.storeContactDetails(False):
raise Exception(psycopg2.DatabaseError)
if self.facebook_url is not None:
contactDetails_facebook = ContactDetails(
"agent_facebook_url", OBJECT_Agent, self.agent_id,
self.facebook_url)
#Attempt to save the facebook details
if not contactDetails_facebook.storeContactDetails(False):
raise Exception(psycopg2.DatabaseError)
if self.twitter_url is not None:
contactDetails_twitter = ContactDetails(
"agent_twitter_url", OBJECT_Agent, self.agent_id,
self.twitter_url)
#Attempt to save the twitter details
if not contactDetails_twitter.storeContactDetails(False):
raise Exception(psycopg2.DatabaseError)
if self.photo is not None:
#Save the mugshot and agent photo.
file_agentPhoto = File(FILE_TYPE_Images, OBJECT_Agent,
self.agent_id, None, "agent_photo")
file_agentPhoto.addImageDetails(None, None, self.photo)
if not file_agentPhoto.storeFile(False):
raise Exception(psycopg2.DatabaseError)
if self.mugshot_url is not None:
file_agentMugShot = File(FILE_TYPE_Images, OBJECT_Agent,
self.agent_id, None, "agent_mugshot")
file_agentMugShot.addImageDetails(None, None, self.mugshot_url)
if not file_agentMugShot.storeFile(False):
raise Exception(psycopg2.DatabaseError)
if commit:
conn.commit()
return self.agent_id
except (Exception, psycopg2.DatabaseError) as error:
print("Error in INSERTING New Agent " + self.first_name + " " +
self.last_name + "\n" + error)
return None
def main(interactions=False,
download=True,
parse=True,
withdrawn=True,
cleanup=True):
"""Performs the download of interaction and annotation files from MGI.
Builds a gene annotation file and mapping tables.
TODO:
- Inspect and eventually use interaction file, else discard from this module.
- Also check whether other information from MGI is worse to integrate
such as homology or phenotypes."""
os.chdir(path)
genes.name = 'MGI'
genes.key = 'mgi'
folder = Folder(path)
if interactions:
ftp = FTP(
url='ftp://ftp.informatics.jax.org/pub/protein-interaction-data/',
path=path)
ftp.download(path)
if download:
url = "ftp://ftp.informatics.jax.org/pub/reports/"
files = [
"MRK_List1.rpt", "MRK_List2.rpt", "MGI_Coordinate.rpt",
"MRK_Sequence.rpt", "MRK_SwissProt_TrEMBL.rpt", "MRK_VEGA.rpt",
"MRK_ENSEMBL.rpt", "MGI_EntrezGene.rpt"
]
# MPheno_OBO.ontology, VOC_MammalianPhenotype.rpt, MGI_PhenotypicAllele.rpt, HMD_HumanPhenotype.rpt
for f in files:
f = File(url=url + f) # automatically does f.download()
res = f.parse(header=True, printing=False)
folder.downloads.append(f.name)
if parse:
folder.update()
if withdrawn: filename = "MRK_List1.rpt"
else: filename = "MRK_List2.rpt"
data = folder[filename].parse(header=True, printing=False)
genes.addData(data, key='mgi', taxid=10090)
data = folder["MGI_Coordinate.rpt"].parse(header=True, printing=False)
for i in data:
i = change_keys(i)
i['taxid'] = 10090
genes.add(i)
data = folder['MRK_Sequence.rpt'].parse(header=True, printing=False)
genes.addData(data, key='mgi', taxid=10090)
header = "mgi symbol status name cm_position chromosome type "\
"secondary_accession_ids id synonyms feature_types start "\
"stop strand biotypes".split()
data = folder["MGI_EntrezGene.rpt"].parse(header=header,
printing=False)
genes.addData(data, key="mgi", taxid=10090)
print len(genes)
if cleanup:
if interactions: ftp.remove(confirm=False)
for f in folder.downloads:
folder.remove(f)
genes.keep("category", "Gene")
genes.remove("name", "withdrawn")
genes.save()
genes.buildMappings()
if commands:
for command in commands:
output, error = c.run_command(command, host)
if error != []:
raise Exception(error[0])
else:
print("\n*** Output of '{}' on '{}'".format(command, host))
for o in output:
print(o.strip('\n'))
elif install_pkgs:
p = Packages()
p.install_package(host, install_pkgs)
s = Service()
s.manageService(pkg_service_deps, 'restart', service_pkg_deps, host)
elif uninstall_pkgs:
p = Packages()
p.uninstall_package(host, uninstall_pkgs)
elif upgrade_pkgs:
p = Packages()
p.upgrade_package(host, upgrade_pkgs)
elif file_data and file_metadata:
f = File()
f.create_file(host, file_data, file_metadata)
elif not file_data and file_metadata:
f = File()
f.delete_file(host, file_metadata)
elif services:
s.manageService(services, service_action, service_pkg_deps, host)
except Exception as e:
print(e)
def __init__(self, path):
"""Read and parse data from file at *path*."""
yaml_string = File(path).read_text()
super().__init__(yaml_string)
