def load_button_pixbufs(color): global BUTTONS_SVG if BUTTONS_SVG is None: image_path = os.path.join(MODULE_DIR, 'images', 'mouse.svg') with open(image_path) as svg_file: BUTTONS_SVG = svg_file.readlines() if not isinstance(color, str): # Gdk.Color color = 'rgb({}, {}, {})'.format(round(color.red_float * 255), round(color.green_float * 255), round(color.blue_float * 255)) button_pixbufs = [] svg = NamedTemporaryFile(mode='w', suffix='.svg') for line in BUTTONS_SVG[1:-1]: svg.seek(0) svg.truncate() svg.writelines(( BUTTONS_SVG[0], line.replace('#fff', color), BUTTONS_SVG[-1], )) svg.flush() os.fsync(svg.fileno()) button_pixbufs.append(GdkPixbuf.Pixbuf.new_from_file(svg.name)) svg.close() return button_pixbufs
def __init__(self, video_generators, video_format): from textwrap import dedent from tempfile import NamedTemporaryFile from subprocess import CalledProcessError, check_output, STDOUT from random import randint self.lighttpd_pid = None self.video_generators = dict(video_generators) video_cache_dir = _gen_video_cache_dir() mkdir_p(video_cache_dir) lighttpd_config_file = NamedTemporaryFile( prefix='stbt-camera-lighttpd-', suffix='.conf', delete=False) pidfile = NamedTemporaryFile(prefix="stbt-camera-lighttpd-", suffix=".pidfile") # This is an awful way to start listening on a random port and not a # great way of tracking the sub-process. port = None while port is None: try: lighttpd_config_file.seek(0) lighttpd_config_file.truncate(0) try_port = randint(10000, 30000) lighttpd_config_file.write( dedent("""\ # This file is generated automatically by stb-tester. # DO NOT EDIT. server.document-root = "%s" server.port = %i server.pid-file = "%s" mimetype.assign = ( ".png" => "image/png", ".mp4" => "video/mp4", ".ts" => "video/MP2T" )""") % (video_cache_dir, try_port, pidfile.name)) lighttpd_config_file.flush() check_output(['lighttpd', '-f', lighttpd_config_file.name], close_fds=True, stderr=STDOUT) port = try_port except CalledProcessError as e: if e.output.find('Address already in use') != -1: pass else: sys.stderr.write("lighttpd failed to start: %s\n" % e.output) raise # lighttpd writes its pidfile out after forking rather than before # causing a race. The real fix is to patch lighttpd to support socket # passing and then open the listening socket ourselves. while os.fstat(pidfile.fileno()).st_size == 0: sleep(0.1) self.lighttpd_pid = int(pidfile.read()) self.base_url = "http://%s:%i/" % (_get_external_ip(), port) self.video_format = video_format
def __init__(self, video_generators, video_format): from textwrap import dedent from tempfile import NamedTemporaryFile from subprocess import CalledProcessError, check_output, STDOUT from random import randint self.lighttpd_pid = None self.video_generators = dict(video_generators) video_cache_dir = _gen_video_cache_dir() mkdir_p(video_cache_dir) lighttpd_config_file = NamedTemporaryFile( prefix='stbt-camera-lighttpd-', suffix='.conf', delete=False) pidfile = NamedTemporaryFile( prefix="stbt-camera-lighttpd-", suffix=".pidfile") # This is an awful way to start listening on a random port and not a # great way of tracking the sub-process. port = None while port is None: try: lighttpd_config_file.seek(0) lighttpd_config_file.truncate(0) try_port = randint(10000, 30000) lighttpd_config_file.write(dedent("""\ # This file is generated automatically by stb-tester. # DO NOT EDIT. server.document-root = "%s" server.port = %i server.pid-file = "%s" mimetype.assign = ( ".png" => "image/png", ".mp4" => "video/mp4", ".ts" => "video/MP2T" )""") % (video_cache_dir, try_port, pidfile.name)) lighttpd_config_file.flush() check_output(['lighttpd', '-f', lighttpd_config_file.name], close_fds=True, stderr=STDOUT) port = try_port except CalledProcessError as e: if e.output.find('Address already in use') != -1: pass else: sys.stderr.write("lighttpd failed to start: %s\n" % e.output) raise # lighttpd writes its pidfile out after forking rather than before # casuing a race. The real fix is to patch lighttpd to support socket # passing and then open the listening socket ourselves. while os.fstat(pidfile.fileno()).st_size == 0: sleep(0.1) self.lighttpd_pid = int(pidfile.read()) self.base_url = "http://%s:%i/" % (_get_external_ip(), port) self.video_format = video_format
def convert_to_pdf(file: NamedTemporaryFile, mime: str) -> bool: '''Convert a file to PDF if necessary. Return whether the conversion took place.''' if mime == 'application/pdf': return False unoconv = subprocess.run(['unoconv', '--stdout', '-f', 'pdf', file.name], text=False, capture_output=True, timeout=60, check=True) file.seek(0) file.write(unoconv.stdout) file.truncate() return True
def process_askue(): e = Exporter() try: logging.debug('Trying to connect to FTP server...') with FTP(S.FTP_SERVER, S.FTP_USER, S.FTP_PASSWORD, timeout=5) as fc: logging.debug('Looking for files in FTP directory') # Find files and retrieve it inbox_files = fc.mlsd(S.REMS_PATH) filenames = [e[0] for e in inbox_files if askue_filename(e[0])] if not filenames: logging.info('Inbox directory is empty...') return if len(filenames) > 1: logging.debug( 'More than 1 file were found: {}'.format( '\n'.join(filenames))) rfile = max(filenames, key=date_from_filename) logging.info('Retrieving {}...'.format(rfile)) tf = NamedTemporaryFile() fc.retrbinary('RETR {}'.format(j(S.REMS_PATH, rfile)), tf.write) ftp_pos = tf.tell() try: if S.APPEND_ON: lines = (record_to_csv(rec) for rec in e.get_routes(datetime.now())) append_lines(tf, lines) else: logging.debug( 'Will not append lines (switched off in settings)') except Exception: logging.exception( 'Error appending lines to file! Sending as is') tf.seek(ftp_pos) tf.truncate() tf.seek(0) dest_path = j(S.IOMM_PATH, rfile) # Send file back to FTP logging.info('Sending file... {}'.format(dest_path)) fc.storbinary('STOR {}'.format(dest_path), tf) logging.info('Cleaning up directory...') for fname in filenames: filepath = j(S.REMS_PATH, fname) fc.delete(filepath) finally: e.close_connection()
def write(data, path): with ZipFile(path, 'w', zipfile.ZIP_DEFLATED) as zip: content = io.StringIO() content.write('Manifest-Version: 1.0\n') content.write('Created-By: jamovi\n') content.write('Data-Archive-Version: 1.0.2\n') content.write('jamovi-Archive-Version: 1.0\n') zip.writestr('META-INF/MANIFEST.MF', bytes(content.getvalue(), 'utf-8'), zipfile.ZIP_DEFLATED) content = None fields = [ ] for column in data.dataset: field = { } field['name'] = column.name field['measureType'] = MeasureType.stringify(column.measure_type) if column.measure_type == MeasureType.CONTINUOUS: field['type'] = 'number' else: field['type'] = 'integer' field['importName'] = column.import_name fields.append(field) metadata = { } metadataset = { } metadataset['rowCount'] = data.dataset.row_count metadataset['columnCount'] = data.dataset.column_count metadataset['fields'] = fields metadata['dataSet'] = metadataset zip.writestr('metadata.json', json.dumps(metadata), zipfile.ZIP_DEFLATED) metadata = None xdata = { } for column in data.dataset: if column.has_levels: xdata[column.name] = { 'labels': column.levels } zip.writestr('xdata.json', json.dumps(xdata), zipfile.ZIP_DEFLATED) xdata = None row_count = data.dataset.row_count required_bytes = 0 for column in data.dataset: if column.measure_type == MeasureType.CONTINUOUS: required_bytes += (8 * row_count) else: required_bytes += (4 * row_count) temp_file = NamedTemporaryFile(delete=False) temp_file.truncate(required_bytes) for column in data.dataset: if column.measure_type == MeasureType.CONTINUOUS: for i in range(0, row_count): value = column.raw(i) byts = struct.pack('<d', value) temp_file.write(byts) else: for i in range(0, row_count): value = column.raw(i) byts = struct.pack('<i', value) temp_file.write(byts) temp_file.close() zip.write(temp_file.name, 'data.bin') os.remove(temp_file.name) resources = [ ] for analysis in data.analyses: if analysis.has_results is False: continue analysis_dir = '{:02} {}/analysis'.format(analysis.id, analysis.name) zip.writestr(analysis_dir, analysis.serialize(), zipfile.ZIP_DEFLATED) resources += analysis.resources for rel_path in resources: abs_path = os.path.join(data.instance_path, rel_path) zip.write(abs_path, rel_path)
def generate_blank_syslinux(): # Generate syslinux.qcow2 in working directory if it isn't already there if os.path.isfile("./syslinux.qcow2"): print "Found a syslinux.qcow2 image in the working directory - using it" return print "Generating an empty bootable syslinux image as ./syslinux.qcow2" raw_fs_image = NamedTemporaryFile(delete=False) raw_image_name = raw_fs_image.name try: output_image_name = "./syslinux.qcow2" # 200 MB sparse file outsize = 1024 * 1024 * 200 raw_fs_image.truncate(outsize) raw_fs_image.close() # Partition, format and add DOS MBR g = guestfs.GuestFS() g.add_drive(raw_image_name) g.launch() g.part_disk("/dev/sda","msdos") g.part_set_mbr_id("/dev/sda",1,0xb) g.mkfs("vfat", "/dev/sda1") g.part_set_bootable("/dev/sda", 1, 1) dosmbr = open("/usr/share/syslinux/mbr.bin").read() ws = g.pwrite_device("/dev/sda", dosmbr, 0) if ws != len(dosmbr): raise Exception("Failed to write entire MBR") g.sync() g.close() # Install syslinux - this is the ugly root-requiring part gotloop = False for n in range(4): # If this has a nonzero return code we will take the exception (stdout, stderr, retcode) = subprocess_check_output(["losetup","-f"]) loopdev = stdout.rstrip() # Race - Try it a few times and then give up try: subprocess_check_output(["losetup",loopdev,raw_image_name]) except: sleep(1) continue gotloop = True break if not gotloop: raise Exception("Failed to setup loopback") loopbase = os.path.basename(loopdev) try: subprocess_check_output(["kpartx","-a",loopdev]) # On RHEL6 there seems to be a short delay before the mappings actually show up sleep(5) subprocess_check_output(["syslinux", "/dev/mapper/%sp1" % (loopbase)]) subprocess_check_output(["kpartx", "-d", loopdev]) subprocess_check_output(["losetup", "-d", loopdev]) except: print "Exception while executing syslinux install commands." raise try: subprocess_check_output(["qemu-img","convert","-c","-O","qcow2",raw_image_name,output_image_name]) except: print "Exception while converting image to qcow2" finally: pass
def main(directory, consumer_key, consumer_secret, access_token_key, access_token_secret): subprocess.call(["xnview", directory]) images = [x for x in os.listdir(directory) if x.endswith(b".jpg")] if not images: return collages = os.path.join(directory, b"collages") if not os.path.exists(collages): os.mkdir(collages) for i in itertools.count(): collage_dir = os.path.join(collages, b"collage%04d" % i) if not os.path.exists(collage_dir): os.mkdir(collage_dir) break collage_path = os.path.join(collage_dir, b"collage.jpg") if len(images) == 1: shutil.copy(os.path.join(directory, images[0]), collage_path) else: bigw, bigh = Image.open(os.path.join(directory, images[0])).size w, h = map(lambda x: int(x / 2), (bigw, bigh)) hborder = int(w * 0.05) vborder = int(h * 0.05) w -= int(hborder / 2) h -= int(vborder / 2) cw = (w + hborder) * 2 + hborder ch = (h + vborder) * (int(len(images) / 2)) + vborder if len(images) % 2: ch += bigh + vborder im = Image.new("RGB", (cw, ch), "black") for i, image in enumerate(images): im2 = Image.open(os.path.join(directory, image)) if len(images) % 2 and i == len(images) - 1: im.paste(im2, (hborder, vborder + int(len(images) / 2) * (h + vborder))) else: im2.thumbnail((w, h), Image.ANTIALIAS) im.paste(im2, (hborder + (w + hborder) * (i % 2), vborder + int(i / 2) * (h + vborder))) im.save(collage_path) subprocess.call(["xnview", collage_path]) f = NamedTemporaryFile() f.write(os.path.basename(directory)) f.flush() while True: try: tweet = subprocess.check_output(["zenity", "--title", "Tweet", "--text-info", "--editable", "--filename", f.name, "--width", "1280", "--height", "320", "--font", "Segoe UI 35"]) except: return decoded = "" try: decoded = tweet.decode("utf-8") twitter.Api(consumer_key=consumer_key, consumer_secret=consumer_secret, access_token_key=access_token_key, access_token_secret=access_token_secret).PostMedia(decoded, collage_path) except Exception as e: subprocess.check_call(["zenity", "--error", "--no-markup", "--text", repr(e) + b" (len=%d)" % len(decoded)]) f.truncate(0) f.seek(0) f.write(tweet) f.flush() else: break for image in images: shutil.move(os.path.join(directory, image), collage_dir)
def _resize_partition_on_disk_image(self, additional_bytes): # This function expects the partitions to be in disk order: it will # fail if there are primary partitions after an extended one containing # logical partitions. # Resizing logical partitions that are not the last on the disk is not # implemented assert self.parent is None partition_table = self.partition_table image = self.image # Create a new temporary file of the correct size tmp = NamedTemporaryFile(dir=os.path.dirname(image), delete=False) tmp.truncate(file_size(image) + additional_bytes) tmp.close() # Update the partition table additional_sectors = additional_bytes // SECTOR_SIZE # resize the partition self.size += additional_sectors # move the partitions after for part in partition_table.partitions[self.number:]: part.start += additional_sectors # update last lba if partition_table.lastlba is not None: partition_table.lastlba += additional_sectors sfdisk(tmp.name, _in=partition_table.get_sfdisk_script(), **SH_OPTS) # Now we copy the data from the image to the temporary file copy = partial( ddd, _if=image, of=tmp.name, bs=1024**2, # one MiB conv="notrunc", iflag="count_bytes,skip_bytes", # count and skip in bytes oflag="seek_bytes", # seek in bytes ) # Preserve GRUB copy(count=MBR_BOOTSTRAP_CODE_SIZE) # Copy across any data that's located between the MBR and the first # partition (some devices rely on the bootloader being there, like the # Variscite DART-6UL) if self.partition_table.label == "dos": copy( skip=MBR_SIZE, seek=MBR_SIZE, count=partition_table.partitions[0].start_bytes - MBR_SIZE, ) elif self.partition_table.label == "gpt": copy( skip=GPT_SIZE, seek=GPT_SIZE, count=partition_table.partitions[0].start_bytes - GPT_SIZE, ) # TODO: if we copy an extended partition, there is no need to copy its # logical partitions. # Copy partitions before and the partition itself for part in partition_table.partitions[:self.number]: # No need to copy extended partitions, we'll copy their logical # partitions if not part.is_extended(): copy( skip=part.start_bytes, seek=part.start_bytes, count=part.size_bytes, ) # Copy partitions after. for part in partition_table.partitions[self.number:]: if not part.is_extended(): copy( skip=part.start_bytes, seek=part.start_bytes + additional_bytes, count=part.size_bytes, ) # Replace the original image contents. ddd(_if=tmp.name, of=image, bs=1024**2)
def build_summary(doc, rouge_settings): params = build_rouge_params(rouge_settings) match_pattern = "X ROUGE-{} Eval".format(rouge_settings["order"]) ref_file = get_reference_file(doc) input_sentences = get_input_sentences(doc, 25) n_inputs = len(input_sentences) input_ids = [i for i in xrange(n_inputs)] candidate_files = [NamedTemporaryFile("w", delete=False) for i in xrange(n_inputs)] config_lines = ["{} {}".format(cf.name, ref_file.name) for cf in candidate_files] config_file = NamedTemporaryFile("w", delete=False) greedy_summary = "" greedy_score = 0 for z in range(n_inputs): cfg_text = "\n".join(config_lines) config_file.truncate(len(cfg_text)) config_file.seek(0) config_file.write(cfg_text) config_file.flush() for i in xrange(len(config_lines)): input_id = input_ids[i] sum = "{}{}\n".format(greedy_summary, input_sentences[input_id]) cf = candidate_files[i] cf.truncate(len(sum)) cf.seek(0) cf.write(sum) cf.flush() output = subprocess.check_output(params + [config_file.name]) i = 0 max_score = greedy_score max_id = None for line in output.split("\n"): if line.startswith(match_pattern): score = float(line.split()[4][2:]) if score > max_score: max_score = score max_id = i i += 1 if max_id is not None: greedy_score = max_score greedy_summary = "{}{}\n".format( greedy_summary, input_sentences[max_id]) input_ids.pop(max_id) config_lines = config_lines[:-1] else: break for cf in candidate_files: cf.close() os.remove(cf.name) os.remove(ref_file.name) config_file.close() os.remove(config_file.name) return greedy_summary.strip()
def download_rules(self): # Rules can be downloaded a couple of ways: # - From a GraphQL supplying the rule body in rawRule # - A file containing either compiled or plain text rule(s) from tempfile import NamedTemporaryFile from json import dumps as json_dumps import shutil urllib3.disable_warnings() # File into which the rules are saved self.temp_rules_file = NamedTemporaryFile(mode='wb') if self.rules_location.endswith('.yar'): # Assume a rules file and download it response = requests.get(self.rules_location, verify=self.verify, cookies=self.cookie) if response.status_code != 200: self.logger.error("%s: %s", response.status_code, response.text) sys.exit(1) self.temp_rules_file.write(response.content) else: # get rules from GraphQL backend, ignoring any disabled ones rules_query = """ query { rules(condition: {isDisabled: false}) { nodes { id name rawRule } } }""" response = requests.post(self.rules_location, json={'query': rules_query}, verify=self.verify, cookies=self.cookie) if response.status_code != 200: self.logger.error("%s: %s", response.status_code, response.text) sys.exit(1) data = response.json() self.logger.debug(json_dumps(data, indent=2)) # Do a quick test run of each rule and ignore any that cause errors or warnings rule_testing_file = NamedTemporaryFile(mode='wb') exclude_rules = ['url', 'with_sqlite'] # too spammy good_rules, problem_rules = {}, {} for rule in data['data']['rules']['nodes']: if any([er == rule['name'] for er in exclude_rules]): problem_rules[rule['name']] = rule['id'] continue if not rule['rawRule']: problem_rules[rule['name']] = rule['id'] continue # Write rule to a temporary file and run it to make sure it runs without error rule_testing_file.seek(0) rule_testing_file.write(rule['rawRule'].encode('utf-8')) rule_testing_file.truncate() rule_testing_file.flush() cmd = "nice -n %d %s --fail-on-warnings -p 1 -f %s /dev/null" % \ (self.nice, self.yara_binary, rule_testing_file.name) try: output = subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT) self.logger.debug("Rule '%s' ok, adding", rule['name']) good_rules[rule['name']] = rule['id'] self.temp_rules_file.write(rule['rawRule'].encode('utf-8')) except subprocess.CalledProcessError as e: self.logger.warning("Rule '%s' failed: '%s', skipping ...", rule['name'], e.output) problem_rules[rule['name']] = rule['id'] continue self.logger.debug("Rules to use ...") self.logger.debug(good_rules) self.logger.debug("Rules that won't be used ...") self.logger.debug(problem_rules) self.temp_rules_file.flush() # if -S/--save option used, copy the temp file to the user specified file and use that file if self.rules_file: self.logger.info("Saving rules to %s", self.rules_file) shutil.copy(self.temp_rules_file.name, self.rules_file) self.temp_rules_file.close() else: self.rules_file = self.temp_rules_file.name
def write(data, path): with ZipFile(path, "w", zipfile.ZIP_DEFLATED) as zip: content = io.StringIO() content.write("Manifest-Version: 1.0\n") content.write("Created-By: JASP 0.7.5 Beta 2\n") content.write("Data-Archive-Version: 1.0.2\n") content.write("JASP-Archive-Version: 2.0\n") zip.writestr("META-INF/MANIFEST.MF", bytes(content.getvalue(), "utf-8"), zipfile.ZIP_DEFLATED) content = None fields = [] for column in data.dataset: field = {} field["name"] = column.name field["measureType"] = MeasureType.stringify(column.measure_type) if column.measure_type == MeasureType.CONTINUOUS: field["type"] = "number" else: field["type"] = "integer" fields.append(field) metadata = {} metadataset = {} metadataset["rowCount"] = data.dataset.row_count metadataset["columnCount"] = data.dataset.column_count metadataset["fields"] = fields metadata["dataSet"] = metadataset zip.writestr("metadata.json", json.dumps(metadata), zipfile.ZIP_DEFLATED) metadata = None xdata = {} for column in data.dataset: if column.has_levels: xdata[column.name] = {"labels": column.levels} zip.writestr("xdata.json", json.dumps(xdata), zipfile.ZIP_DEFLATED) xdata = None row_count = data.dataset.row_count required_bytes = 0 for column in data.dataset: if column.measure_type == MeasureType.CONTINUOUS: required_bytes += 8 * row_count else: required_bytes += 4 * row_count temp_file = NamedTemporaryFile(delete=False) temp_file.truncate(required_bytes) for column in data.dataset: if column.measure_type == MeasureType.CONTINUOUS: for i in range(0, row_count): value = column.raw(i) byts = struct.pack("<d", value) temp_file.write(byts) else: for i in range(0, row_count): value = column.raw(i) byts = struct.pack("<i", value) temp_file.write(byts) temp_file.close() zip.write(temp_file.name, "data.bin") os.remove(temp_file.name)
def write(data, path, prog_cb, html=None, is_template=False): with ZipFile(path, 'w', zipfile.ZIP_DEFLATED) as zip: content = io.StringIO() content.write('Manifest-Version: 1.0\n') content.write('Data-Archive-Version: 1.0.2\n') content.write('jamovi-Archive-Version: 9.0\n') content.write('Created-By: ' + str(app_info) + '\n') zip.writestr('META-INF/MANIFEST.MF', bytes(content.getvalue(), 'utf-8'), zipfile.ZIP_DEFLATED) if html is not None: zip.writestr('index.html', html) content = None string_table_required = False transforms = [] for transform in data.transforms: transform_field = {} transform_field['name'] = transform.name transform_field['id'] = transform.id transform_field['suffix'] = transform.suffix transform_field['formula'] = transform.formula transform_field['formulaMessage'] = transform.formula_message transform_field['measureType'] = MeasureType.stringify( transform.measure_type) transform_field['description'] = transform.description transforms.append(transform_field) fields = [] for column in data: if column.is_virtual is True: continue field = {} field['name'] = column.name field['id'] = column.id field['columnType'] = ColumnType.stringify(column.column_type) field['dataType'] = DataType.stringify(column.data_type) field['measureType'] = MeasureType.stringify(column.measure_type) field['formula'] = column.formula field['formulaMessage'] = column.formula_message field['parentId'] = column.parent_id field['width'] = column.width if column.data_type == DataType.DECIMAL: field['type'] = 'number' elif column.data_type == DataType.TEXT and column.measure_type == MeasureType.ID: field['type'] = 'string' string_table_required = True else: field['type'] = 'integer' if column.column_type is ColumnType.OUTPUT: field['outputAnalysisId'] = column.output_analysis_id field['outputOptionName'] = column.output_option_name field['outputName'] = column.output_name field[ 'outputDesiredColumnName'] = column.output_desired_column_name field[ 'outputAssignedColumnName'] = column.output_assigned_column_name field['importName'] = column.import_name field['description'] = column.description field['transform'] = column.transform field['edits'] = column.cell_tracker.edited_cell_ranges field['missingValues'] = column.missing_values if column.is_filter: field['filterNo'] = column.filter_no # field['hidden'] = column.hidden field['active'] = column.active else: if column.has_levels: field['trimLevels'] = column.trim_levels fields.append(field) if is_template: row_count = 0 else: row_count = data.row_count metadata = {} metadataset = {} metadataset['rowCount'] = row_count metadataset['columnCount'] = data.column_count metadataset['removedRows'] = data.row_tracker.removed_row_ranges metadataset['addedRows'] = data.row_tracker.added_row_ranges metadataset['fields'] = fields metadataset['transforms'] = transforms # if data.import_path is not '': # metadataset['importPath'] = data.import_path # if data.embedded_path is not '': # metadataset['embeddedPath'] = data.embedded_path # if data.embedded_name is not '': # metadataset['embeddedName'] = data.embedded_name metadata['dataSet'] = metadataset zip.writestr('metadata.json', json.dumps(metadata), zipfile.ZIP_DEFLATED) metadata = None xdata = {} for column in data: if column.is_virtual is True: continue if column.has_levels: xdata[column.name] = {'labels': column.levels} zip.writestr('xdata.json', json.dumps(xdata), zipfile.ZIP_DEFLATED) xdata = None required_bytes = 0 for column in data: if column.is_virtual is True: continue if column.data_type == DataType.DECIMAL: required_bytes += (8 * row_count) else: required_bytes += (4 * row_count) if string_table_required: cursor = 0 string_file = NamedTemporaryFile(delete=False) temp_file = NamedTemporaryFile(delete=False) temp_file.truncate(required_bytes) for col_no in range(data.column_count): column = data[col_no] if column.is_virtual is True: continue if column.data_type == DataType.DECIMAL: for i in range(0, row_count): value = column.raw(i) byts = struct.pack('<d', value) temp_file.write(byts) if i % 100000 == 0: prog_cb((col_no + i / row_count) / data.column_count) elif column.data_type == DataType.TEXT and column.measure_type == MeasureType.ID: for i in range(0, row_count): value = column[i] if value != '': byts = value.encode('utf-8') string_file.write(byts) string_file.write(bytes(1)) n = len(byts) + 1 byts = struct.pack('<i', cursor) temp_file.write(byts) cursor += n else: byts = struct.pack('<i', -2147483648) temp_file.write(byts) if i % 100000 == 0: prog_cb((col_no + i / row_count) / data.column_count) else: for i in range(0, row_count): value = column.raw(i) byts = struct.pack('<i', value) temp_file.write(byts) if i % 100000 == 0: prog_cb((col_no + i / row_count) / data.column_count) temp_file.close() zip.write(temp_file.name, 'data.bin') os.remove(temp_file.name) if string_table_required: string_file.close() zip.write(string_file.name, 'strings.bin') os.remove(string_file.name) resources = [] for analysis in data.analyses: if analysis.has_results is False: continue analysis_dir = '{:02} {}/analysis'.format(analysis.id, analysis.name) zip.writestr(analysis_dir, analysis.serialize(strip_content=is_template), zipfile.ZIP_DEFLATED) resources += analysis.resources for rel_path in resources: abs_path = os.path.join(data.instance_path, rel_path) zip.write(abs_path, rel_path)
def o4_pyforce(debug, no_revision, args: list, quiet=False): """ Encapsulates Pyforce, does book keeping to ensure that all files that should be operated on are in fact dealt with by p4. Handles retry and strips out asks for files that are caseful mismatches on the current file system (macOS). """ from tempfile import NamedTemporaryFile from collections import defaultdict class LogAndAbort(Exception): 'Dumps debug information on errors.' o4_log('pyforce', no_revision=no_revision, quiet=quiet, *args) tmpf = NamedTemporaryFile(dir='.o4') fstats = [] for line in sys.stdin.read().splitlines(): if line.startswith('#o4pass'): print(line) continue f = fstat_split(line) if f and caseful_accurate(f[F_PATH]): fstats.append(f) elif f: print( f"*** WARNING: Pyforce is skipping {f[F_PATH]} because it is casefully", "mismatching a local file.", file=sys.stderr) retries = 3 head = _depot_path().replace('/...', '') while fstats: if no_revision: p4paths = [Pyforce.escape(f[F_PATH]) for f in fstats] else: p4paths = [ f"{Pyforce.escape(f[F_PATH])}#{f[F_REVISION]}" for f in fstats ] tmpf.seek(0) tmpf.truncate() not_yet = [] pargs = [] xargs = [] # This is a really bad idea, files are output to stdout before the actual # sync happens, causing checksum tests to start too early: # if len(p4paths) > 30 and 'sync' in args: # xargs.append('--parallel=threads=5') if sum(len(s) for s in p4paths) > 30000: pargs.append('-x') pargs.append(tmpf.name) for f in p4paths: tmpf.write(f.encode('utf8')) tmpf.write(b'\n') tmpf.flush() else: xargs.extend(p4paths) try: # TODO: Verbose #print('# PYFORCE({}, {}{})'.format(','.join(repr(a) for a in args), ','.join( # repr(a) for a in paths[:3]), ', ...' if len(paths) > 3 else '')) errs = [] repeats = defaultdict(list) infos = [] for res in Pyforce(*pargs, *args, *xargs): if debug: err_print("*** DEBUG: Received", repr(res)) # FIXME: Delete this if-statement: if res.get('code', '') == 'info': infos.append(res) if res.get('data', '').startswith('Diff chunks: '): continue if res.get('code', '') == 'error': errs.append(res) continue if 'resolveFlag' in res: # TODO: resolveFlag can be ...? # m: merge # c: copy from (not conflict!) # We skip this entry as it is the second returned from p4 # for one input file continue res_str = res.get('depotFile') or res.get('fromFile') if not res_str and res.get('data'): res_str = head + '/' + res['data'] if not res_str: errs.append(res) continue res_str = Pyforce.unescape(res_str) for i, f in enumerate(fstats): if f"{head}/{f[F_PATH]}" in res_str: repeats[f"{head}/{f[F_PATH]}"].append(res) not_yet.append(fstats.pop(i)) break else: for f in repeats.keys(): if f in res_str: if debug: err_print( f"*** DEBUG: REPEAT: {res_str}\n {res}\n {repeats[f]}" ) break else: if debug: err_print("*** DEBUG: ERRS APPEND", res) errs.append(res) if errs: raise LogAndAbort('Unexpected reply from p4') if len(p4paths) == len(fstats): raise LogAndAbort('Nothing recognized from p4') except P4Error as e: non_recoverable = False for a in e.args: if 'clobber writable file' in a['data']: fname = a['data'].split('clobber writable file')[1].strip() print("*** WARNING: Saving writable file as .bak:", fname, file=sys.stderr) if os.path.exists(fname + '.bak'): now = time.time() print( f"*** WARNING: Moved previous .bak to {fname}.{now}", file=sys.stderr) os.rename(fname + '.bak', f'{fname}.bak.{now}') shutil.copy(fname, fname + '.bak') os.chmod(fname, 0o400) else: non_recoverable = True if non_recoverable: raise except P4TimeoutError as e: e = str(e).replace('\n', ' ') print(f"# P4 TIMEOUT, RETRIES {retries}: {e}", file=sys.stderr) retries -= 1 if not retries: sys.exit( f"{CLR}*** ERROR: Perforce timed out too many times:\n{e}") except LogAndAbort as e: import json fname = f'debug-pyforce.{os.getpid()}.{int(time.time())}' d = { 'args': args, 'fstats': fstats, 'errs': errs, 'repeats': repeats, 'infos': infos, } json.dump(d, open(f'.o4/{fname}', 'wt')) sys.exit(f'{CLR}*** ERROR: {e}; detail in {fname}') finally: if not quiet: for fstat in not_yet: # Printing the fstats after the p4 process has ended, because p4 marshals # its objects before operation, as in "And for my next act... !" # This premature printing leads to false checksum errors during sync. print(fstat_join(fstat))
class File(object): """ A file wrapper that smooths over some platform-specific operations. """ def __init__(self, name=None, readonly=False, **kwargs): if name is None: self.file = NamedTemporaryFile(**kwargs) else: if exists(name): if readonly: self.file = open(name, 'rb') else: self.file = open(name, 'r+b') else: if readonly: raise OSError('No "%s" found.' % name) self.file = open(name, 'w+b') if readonly: assert self.is_readonly() self.has_lock = False def get_name(self): return self.file.name def is_temporary(self): return isinstance(self.file, _TemporaryFileWrapper) def is_readonly(self): return self.file.mode == 'rb' def seek(self, n, whence=0): self.file.seek(n, whence) if whence == 0: assert self.file.tell() == n def seek_end(self): self.file.seek(0, 2) def read(self, n=None): if n is None: return self.file.read() else: return self.file.read(n) def tell(self): return self.file.tell() def stat(self): return os.stat(self.get_name()) def __len__(self): return self.stat().st_size def rename(self, name): old_name = self.get_name() if name == old_name: return assert not self.is_temporary() self.obtain_lock() self.close() if exists(name): os.unlink(name) os.rename(old_name, name) self.file = open(name, 'r+b') self.obtain_lock() def obtain_lock(self): """ Make sure that we have an exclusive lock on self.file before doing a write. If the lock is not available, raise an exception. """ assert not self.is_readonly() if not self.has_lock: if os.name == 'nt': try: win32file.LockFileEx( win32file._get_osfhandle(self.file.fileno()), (win32con.LOCKFILE_EXCLUSIVE_LOCK | win32con.LOCKFILE_FAIL_IMMEDIATELY), 0, -65536, pywintypes.OVERLAPPED()) except pywintypes.error: raise IOError("Unable to obtain lock") else: fcntl.flock(self.file, fcntl.LOCK_EX | fcntl.LOCK_NB) self.has_lock = True def release_lock(self): """ Make sure that we do not retain an exclusive lock on self.file. """ if self.has_lock: if os.name == 'nt': win32file.UnlockFileEx( win32file._get_osfhandle(self.file.fileno()), 0, -65536, pywintypes.OVERLAPPED()) else: fcntl.flock(self.file, fcntl.LOCK_UN) self.has_lock = False def write(self, s): self.obtain_lock() self.file.write(s) if os.name == 'nt': # This flush helps the file knows where it ends. self.file.flush() def truncate(self): self.obtain_lock() self.file.truncate() def close(self): self.release_lock() self.file.close() def flush(self): self.file.flush() def fsync(self): if hasattr(os, 'fsync'): os.fsync(self.file)
def generate_blank_syslinux(): # Generate syslinux.qcow2 in working directory if it isn't already there if os.path.isfile("./syslinux.qcow2"): print "Found a syslinux.qcow2 image in the working directory - using it" return print "Generating an empty bootable syslinux image as ./syslinux.qcow2" raw_fs_image = NamedTemporaryFile(delete=False) raw_image_name = raw_fs_image.name try: output_image_name = "./syslinux.qcow2" # 200 MB sparse file outsize = 1024 * 1024 * 200 raw_fs_image.truncate(outsize) raw_fs_image.close() # Partition, format and add DOS MBR g = guestfs.GuestFS() g.add_drive(raw_image_name) g.launch() g.part_disk("/dev/sda", "msdos") g.part_set_mbr_id("/dev/sda", 1, 0xb) g.mkfs("vfat", "/dev/sda1") g.part_set_bootable("/dev/sda", 1, 1) dosmbr = open("/usr/share/syslinux/mbr.bin").read() ws = g.pwrite_device("/dev/sda", dosmbr, 0) if ws != len(dosmbr): raise Exception("Failed to write entire MBR") g.sync() g.close() # Install syslinux - this is the ugly root-requiring part gotloop = False for n in range(4): # If this has a nonzero return code we will take the exception (stdout, stderr, retcode) = subprocess_check_output(["losetup", "-f"]) loopdev = stdout.rstrip() # Race - Try it a few times and then give up try: subprocess_check_output(["losetup", loopdev, raw_image_name]) except: sleep(1) continue gotloop = True break if not gotloop: raise Exception("Failed to setup loopback") loopbase = os.path.basename(loopdev) try: subprocess_check_output(["kpartx", "-a", loopdev]) # On RHEL6 there seems to be a short delay before the mappings actually show up sleep(5) subprocess_check_output( ["syslinux", "/dev/mapper/%sp1" % (loopbase)]) subprocess_check_output(["kpartx", "-d", loopdev]) subprocess_check_output(["losetup", "-d", loopdev]) except: print "Exception while executing syslinux install commands." raise try: subprocess_check_output([ "qemu-img", "convert", "-c", "-O", "qcow2", raw_image_name, output_image_name ]) except: print "Exception while converting image to qcow2" finally: pass
def write(data, path): with ZipFile(path, 'w', zipfile.ZIP_DEFLATED) as zip: content = io.StringIO() content.write('Manifest-Version: 1.0\n') content.write('Created-By: jamovi\n') content.write('Data-Archive-Version: 1.0.2\n') content.write('jamovi-Archive-Version: 1.0\n') zip.writestr('META-INF/MANIFEST.MF', bytes(content.getvalue(), 'utf-8'), zipfile.ZIP_DEFLATED) content = None fields = [] for column in data.dataset: field = {} field['name'] = column.name field['measureType'] = MeasureType.stringify(column.measure_type) if column.measure_type == MeasureType.CONTINUOUS: field['type'] = 'number' else: field['type'] = 'integer' field['importName'] = column.import_name fields.append(field) metadata = {} metadataset = {} metadataset['rowCount'] = data.dataset.row_count metadataset['columnCount'] = data.dataset.column_count metadataset['fields'] = fields metadata['dataSet'] = metadataset zip.writestr('metadata.json', json.dumps(metadata), zipfile.ZIP_DEFLATED) metadata = None xdata = {} for column in data.dataset: if column.has_levels: xdata[column.name] = {'labels': column.levels} zip.writestr('xdata.json', json.dumps(xdata), zipfile.ZIP_DEFLATED) xdata = None row_count = data.dataset.row_count required_bytes = 0 for column in data.dataset: if column.measure_type == MeasureType.CONTINUOUS: required_bytes += (8 * row_count) else: required_bytes += (4 * row_count) temp_file = NamedTemporaryFile(delete=False) temp_file.truncate(required_bytes) for column in data.dataset: if column.measure_type == MeasureType.CONTINUOUS: for i in range(0, row_count): value = column.raw(i) byts = struct.pack('<d', value) temp_file.write(byts) else: for i in range(0, row_count): value = column.raw(i) byts = struct.pack('<i', value) temp_file.write(byts) temp_file.close() zip.write(temp_file.name, 'data.bin') os.remove(temp_file.name) resources = [] for analysis in data.analyses: if analysis.has_results is False: continue analysis_dir = '{:02} {}/analysis'.format(analysis.id, analysis.name) zip.writestr(analysis_dir, analysis.serialize(), zipfile.ZIP_DEFLATED) resources += analysis.resources for rel_path in resources: abs_path = os.path.join(data.instance_path, rel_path) zip.write(abs_path, rel_path) data.title = os.path.splitext(os.path.basename(path))[0] data.path = path
def write(data, path, prog_cb): with ZipFile(path, 'w', zipfile.ZIP_DEFLATED) as zip: content = io.StringIO() content.write('Manifest-Version: 1.0\n') content.write('Created-By: JASP 0.7.5 Beta 2\n') content.write('Data-Archive-Version: 1.0.2\n') content.write('JASP-Archive-Version: 2.0\n') zip.writestr('META-INF/MANIFEST.MF', bytes(content.getvalue(), 'utf-8'), zipfile.ZIP_DEFLATED) content = None fields = [ ] for column in data.dataset: field = { } field['name'] = column.name field['measureType'] = MeasureType.stringify(column.measure_type) if column.measure_type == MeasureType.CONTINUOUS: field['type'] = 'number' else: field['type'] = 'integer' fields.append(field) metadata = { } metadataset = { } metadataset['rowCount'] = data.dataset.row_count metadataset['columnCount'] = data.dataset.column_count metadataset['fields'] = fields metadata['dataSet'] = metadataset zip.writestr('metadata.json', json.dumps(metadata), zipfile.ZIP_DEFLATED) metadata = None xdata = { } for column in data.dataset: if column.has_levels: xdata[column.name] = { 'labels': column.levels } zip.writestr('xdata.json', json.dumps(xdata), zipfile.ZIP_DEFLATED) xdata = None row_count = data.dataset.row_count required_bytes = 0 for column in data.dataset: if column.measure_type == MeasureType.CONTINUOUS: required_bytes += (8 * row_count) else: required_bytes += (4 * row_count) temp_file = NamedTemporaryFile(delete=False) temp_file.truncate(required_bytes) for column in data.dataset: if column.measure_type == MeasureType.CONTINUOUS: for i in range(0, row_count): value = column.raw(i) byts = struct.pack('<d', value) temp_file.write(byts) else: for i in range(0, row_count): value = column.raw(i) byts = struct.pack('<i', value) temp_file.write(byts) temp_file.close() zip.write(temp_file.name, 'data.bin') os.remove(temp_file.name)
class File (object): """ A file wrapper that smooths over some platform-specific operations. """ def __init__(self, name=None, readonly=False, **kwargs): if name is None: self.file = NamedTemporaryFile(**kwargs) else: if exists(name): if readonly: self.file = open(name, 'rb') else: self.file = open(name, 'r+b') else: if readonly: raise OSError('No "%s" found.' % name) self.file = open(name, 'w+b') if readonly: assert self.is_readonly() self.has_lock = False def get_name(self): return self.file.name def is_temporary(self): return isinstance(self.file, _TemporaryFileWrapper) def is_readonly(self): return self.file.mode == 'rb' def seek(self, n, whence=0): self.file.seek(n, whence) if whence == 0: assert self.file.tell() == n def seek_end(self): self.file.seek(0, 2) def read(self, n=None): if n is None: return self.file.read() else: return self.file.read(n) def tell(self): return self.file.tell() def stat(self): return os.stat(self.get_name()) def __len__(self): return self.stat().st_size def rename(self, name): old_name = self.get_name() if name == old_name: return assert not self.is_temporary() self.obtain_lock() self.close() if exists(name): os.unlink(name) os.rename(old_name, name) self.file = open(name, 'r+b') self.obtain_lock() def obtain_lock(self): """ Make sure that we have an exclusive lock on self.file before doing a write. If the lock is not available, raise an exception. """ assert not self.is_readonly() if not self.has_lock: if os.name == 'nt': try: win32file.LockFileEx( win32file._get_osfhandle(self.file.fileno()), (win32con.LOCKFILE_EXCLUSIVE_LOCK | win32con.LOCKFILE_FAIL_IMMEDIATELY), 0, -65536, pywintypes.OVERLAPPED()) except pywintypes.error: raise IOError("Unable to obtain lock") else: fcntl.flock(self.file, fcntl.LOCK_EX | fcntl.LOCK_NB) self.has_lock = True def release_lock(self): """ Make sure that we do not retain an exclusive lock on self.file. """ if self.has_lock: if os.name == 'nt': win32file.UnlockFileEx( win32file._get_osfhandle(self.file.fileno()), 0, -65536, pywintypes.OVERLAPPED()) else: fcntl.flock(self.file, fcntl.LOCK_UN) self.has_lock = False def write(self, s): self.obtain_lock() self.file.write(s) # This flush helps the file knows where it ends. self.file.flush() def truncate(self): self.obtain_lock() self.file.truncate() def close(self): self.release_lock() self.file.close() def flush(self): self.file.flush() def fsync(self): if hasattr(os, 'fsync'): os.fsync(self.file)
def write(dataset, path): with ZipFile(path, 'w', zipfile.ZIP_DEFLATED) as zip: content = io.StringIO() content.write('Manifest-Version: 1.0\n') content.write('Created-By: JASP 0.7.5 Beta 2\n') content.write('Data-Archive-Version: 1.0.2\n') content.write('JASP-Archive-Version: 2.0\n') zip.writestr('META-INF/MANIFEST.MF', bytes(content.getvalue(), 'utf-8'), zipfile.ZIP_DEFLATED) content = None fields = [ ] for column in dataset: field = { } field['name'] = column.name field['measureType'] = MeasureType.stringify(column.type) if column.type == MeasureType.CONTINUOUS: field['type'] = 'number' else: field['type'] = 'integer' fields.append(field) metadata = { } metadataset = { } metadataset['rowCount'] = dataset.row_count metadataset['columnCount'] = dataset.column_count metadataset['fields'] = fields metadata['dataSet'] = metadataset zip.writestr('metadata.json', json.dumps(metadata), zipfile.ZIP_DEFLATED) metadata = None xdata = { } for column in dataset: if column.has_labels: xdata[column.name] = column.labels zip.writestr('xdata.json', json.dumps(xdata), zipfile.ZIP_DEFLATED) xdata = None row_count = dataset.row_count required_bytes = 0 for column in dataset: if column.type == MeasureType.CONTINUOUS: required_bytes += (8 * row_count) else: required_bytes += (4 * row_count) temp_file = NamedTemporaryFile(delete=False) temp_file.truncate(required_bytes) for column in dataset: if column.type == MeasureType.CONTINUOUS: for i in range(0, row_count): value = column.raw(i) byts = struct.pack('<d', value) temp_file.write(byts) else: for i in range(0, row_count): value = column.raw(i) byts = struct.pack('<i', value) temp_file.write(byts) temp_file.close() zip.write(temp_file.name, 'data.bin') os.remove(temp_file.name)
def write(data, path, html=None): with ZipFile(path, 'w', zipfile.ZIP_DEFLATED) as zip: content = io.StringIO() content.write('Manifest-Version: 1.0\n') content.write('Data-Archive-Version: 1.0.2\n') content.write('jamovi-Archive-Version: 5.0\n') content.write('Created-By: ' + str(app_info) + '\n') zip.writestr('META-INF/MANIFEST.MF', bytes(content.getvalue(), 'utf-8'), zipfile.ZIP_DEFLATED) if html is not None: zip.writestr('index.html', html) content = None string_table_required = False fields = [] for column in data: if column.is_virtual is True: continue field = {} field['name'] = column.name field['columnType'] = ColumnType.stringify(column.column_type) field['dataType'] = DataType.stringify(column.data_type) field['measureType'] = MeasureType.stringify(column.measure_type) field['formula'] = column.formula field['formulaMessage'] = column.formula_message if column.data_type == DataType.DECIMAL: field['type'] = 'number' elif column.data_type == DataType.TEXT and column.measure_type == MeasureType.ID: field['type'] = 'string' string_table_required = True else: field['type'] = 'integer' field['importName'] = column.import_name field['description'] = column.description if column.is_filter: field['filterNo'] = column.filter_no field['hidden'] = column.hidden field['active'] = column.active else: if column.has_levels: field['trimLevels'] = column.trim_levels fields.append(field) metadata = {} metadataset = {} metadataset['rowCount'] = data.row_count metadataset['columnCount'] = data.column_count metadataset['fields'] = fields # if data.import_path is not '': # metadataset['importPath'] = data.import_path # if data.embedded_path is not '': # metadataset['embeddedPath'] = data.embedded_path # if data.embedded_name is not '': # metadataset['embeddedName'] = data.embedded_name metadata['dataSet'] = metadataset zip.writestr('metadata.json', json.dumps(metadata), zipfile.ZIP_DEFLATED) metadata = None xdata = {} for column in data: if column.is_virtual is True: continue if column.has_levels: xdata[column.name] = {'labels': column.levels} zip.writestr('xdata.json', json.dumps(xdata), zipfile.ZIP_DEFLATED) xdata = None row_count = data.row_count required_bytes = 0 for column in data: if column.is_virtual is True: continue if column.data_type == DataType.DECIMAL: required_bytes += (8 * row_count) else: required_bytes += (4 * row_count) if string_table_required: cursor = 0 string_file = NamedTemporaryFile(delete=False) temp_file = NamedTemporaryFile(delete=False) temp_file.truncate(required_bytes) for column in data: if column.is_virtual is True: continue if column.data_type == DataType.DECIMAL: for i in range(0, row_count): value = column.raw(i) byts = struct.pack('<d', value) temp_file.write(byts) elif column.data_type == DataType.TEXT and column.measure_type == MeasureType.ID: for i in range(0, row_count): value = column[i] if value != '': string_file.write(value.encode('utf-8')) string_file.write(bytes(1)) n = len(value) + 1 byts = struct.pack('<i', cursor) temp_file.write(byts) cursor += n else: byts = struct.pack('<i', -2147483648) temp_file.write(byts) else: for i in range(0, row_count): value = column.raw(i) byts = struct.pack('<i', value) temp_file.write(byts) temp_file.close() zip.write(temp_file.name, 'data.bin') os.remove(temp_file.name) if string_table_required: string_file.close() zip.write(string_file.name, 'strings.bin') os.remove(string_file.name) resources = [] for analysis in data.analyses: if analysis.has_results is False: continue analysis_dir = '{:02} {}/analysis'.format(analysis.id, analysis.name) zip.writestr(analysis_dir, analysis.serialize(), zipfile.ZIP_DEFLATED) resources += analysis.resources for rel_path in resources: abs_path = os.path.join(data.instance_path, rel_path) zip.write(abs_path, rel_path)
def create_syslinux_stub(self, image_name, cmdline, kernel_filename, ramdisk_filename): """ @param cmdline: kernel command line @param kernel_filename: path to kernel file @param ramdisk_filename: path to ramdisk file @return glance image id """ raw_fs_image = NamedTemporaryFile(delete=False) raw_image_name = raw_fs_image.name tmp_content_dir = None glance_image_id = None try: qcow2_image_name = "%s.qcow2" % raw_image_name # 200 MB sparse file self.log.debug("Creating sparse 200 MB file") outsize = 1024 * 1024 * 200 raw_fs_image.truncate(outsize) raw_fs_image.close() # Partition, format and add DOS MBR g = guestfs.GuestFS() g.add_drive(raw_image_name) g.launch() g.part_disk("/dev/sda","msdos") g.part_set_mbr_id("/dev/sda",1,0xb) g.mkfs("vfat", "/dev/sda1") g.part_set_bootable("/dev/sda", 1, 1) dosmbr = open("/usr/share/syslinux/mbr.bin").read() ws = g.pwrite_device("/dev/sda", dosmbr, 0) if ws != len(dosmbr): raise Exception("Failed to write entire MBR") # Install syslinux g.syslinux("/dev/sda1") #Insert kernel, ramdisk and syslinux.cfg file tmp_content_dir = mkdtemp() kernel_dest = os.path.join(tmp_content_dir,"vmlinuz") shutil.copy(kernel_filename, kernel_dest) initrd_dest = os.path.join(tmp_content_dir,"initrd.img") shutil.copy(ramdisk_filename, initrd_dest) syslinux_conf="""default customhd timeout 30 prompt 1 label customhd kernel vmlinuz append initrd=initrd.img %s """ % (cmdline) f = open(os.path.join(tmp_content_dir, "syslinux.cfg"),"w") f.write(syslinux_conf) f.close() # copy the tmp content to the image g.mount_options ("", "/dev/sda1", "/") for filename in os.listdir(tmp_content_dir): g.upload(os.path.join(tmp_content_dir,filename),"/" + filename) g.sync() g.close() try: self.log.debug("Converting syslinux stub image from raw to qcow2") self._subprocess_check_output(["qemu-img","convert","-c","-O","qcow2",raw_image_name, qcow2_image_name]) self.log.debug("Uploading syslinux qcow2 image to glance") glance_image_id = self.env.upload_image_to_glance(image_name, local_path=qcow2_image_name, format='qcow2') except Exception, e: self.log.debug("Exception while converting syslinux image to qcow2: %s" % e) self.log.debug("Uploading syslinux raw image to glance.") glance_image_id = self.env.upload_image_to_glance(image_name, local_path=raw_image_name, format='raw') finally: self.log.debug("Removing temporary file.") if os.path.exists(raw_image_name): os.remove(raw_image_name) if os.path.exists(qcow2_image_name): os.remove(qcow2_image_name) if tmp_content_dir: shutil.rmtree(tmp_content_dir) return glance_image_id
class LazyZipOverHTTP: """File-like object mapped to a ZIP file over HTTP. This uses HTTP range requests to lazily fetch the file's content, which is supposed to be fed to ZipFile. If such requests are not supported by the server, raise HTTPRangeRequestUnsupported during initialization. """ def __init__(self, url: str, session: PipSession, chunk_size: int = CONTENT_CHUNK_SIZE) -> None: head = session.head(url, headers=HEADERS) raise_for_status(head) assert head.status_code == 200 self._session, self._url, self._chunk_size = session, url, chunk_size self._length = int(head.headers["Content-Length"]) self._file = NamedTemporaryFile() self.truncate(self._length) self._left: List[int] = [] self._right: List[int] = [] if "bytes" not in head.headers.get("Accept-Ranges", "none"): raise HTTPRangeRequestUnsupported("range request is not supported") self._check_zip() @property def mode(self) -> str: """Opening mode, which is always rb.""" return "rb" @property def name(self) -> str: """Path to the underlying file.""" return self._file.name def seekable(self) -> bool: """Return whether random access is supported, which is True.""" return True def close(self) -> None: """Close the file.""" self._file.close() @property def closed(self) -> bool: """Whether the file is closed.""" return self._file.closed def read(self, size: int = -1) -> bytes: """Read up to size bytes from the object and return them. As a convenience, if size is unspecified or -1, all bytes until EOF are returned. Fewer than size bytes may be returned if EOF is reached. """ download_size = max(size, self._chunk_size) start, length = self.tell(), self._length stop = length if size < 0 else min(start + download_size, length) start = max(0, stop - download_size) self._download(start, stop - 1) return self._file.read(size) def readable(self) -> bool: """Return whether the file is readable, which is True.""" return True def seek(self, offset: int, whence: int = 0) -> int: """Change stream position and return the new absolute position. Seek to offset relative position indicated by whence: * 0: Start of stream (the default). pos should be >= 0; * 1: Current position - pos may be negative; * 2: End of stream - pos usually negative. """ return self._file.seek(offset, whence) def tell(self) -> int: """Return the current position.""" return self._file.tell() def truncate(self, size: Optional[int] = None) -> int: """Resize the stream to the given size in bytes. If size is unspecified resize to the current position. The current stream position isn't changed. Return the new file size. """ return self._file.truncate(size) def writable(self) -> bool: """Return False.""" return False def __enter__(self) -> "LazyZipOverHTTP": self._file.__enter__() return self def __exit__(self, *exc: Any) -> Optional[bool]: return self._file.__exit__(*exc) @contextmanager def _stay(self) -> Iterator[None]: """Return a context manager keeping the position. At the end of the block, seek back to original position. """ pos = self.tell() try: yield finally: self.seek(pos) def _check_zip(self) -> None: """Check and download until the file is a valid ZIP.""" end = self._length - 1 for start in reversed(range(0, end, self._chunk_size)): self._download(start, end) with self._stay(): try: # For read-only ZIP files, ZipFile only needs # methods read, seek, seekable and tell. ZipFile(self) # type: ignore except BadZipfile: pass else: break def _stream_response(self, start: int, end: int, base_headers: Dict[str, str] = HEADERS) -> Response: """Return HTTP response to a range request from start to end.""" headers = base_headers.copy() headers["Range"] = f"bytes={start}-{end}" # TODO: Get range requests to be correctly cached headers["Cache-Control"] = "no-cache" return self._session.get(self._url, headers=headers, stream=True) def _merge(self, start: int, end: int, left: int, right: int) -> Iterator[Tuple[int, int]]: """Return an iterator of intervals to be fetched. Args: start (int): Start of needed interval end (int): End of needed interval left (int): Index of first overlapping downloaded data right (int): Index after last overlapping downloaded data """ lslice, rslice = self._left[left:right], self._right[left:right] i = start = min([start] + lslice[:1]) end = max([end] + rslice[-1:]) for j, k in zip(lslice, rslice): if j > i: yield i, j - 1 i = k + 1 if i <= end: yield i, end self._left[left:right], self._right[left:right] = [start], [end] def _download(self, start: int, end: int) -> None: """Download bytes from start to end inclusively.""" with self._stay(): left = bisect_left(self._right, start) right = bisect_right(self._left, end) for start, end in self._merge(start, end, left, right): response = self._stream_response(start, end) response.raise_for_status() self.seek(start) for chunk in response_chunks(response, self._chunk_size): self._file.write(chunk)
class LazyZipOverHTTP(object): """File-like object mapped to a ZIP file over HTTP. This uses HTTP range requests to lazily fetch the file's content, which is supposed to be fed to ZipFile. If such requests are not supported by the server, raise HTTPRangeRequestUnsupported during initialization. """ def __init__(self, url, session, chunk_size=CONTENT_CHUNK_SIZE): # type: (str, PipSession, int) -> None head = session.head(url, headers=HEADERS) raise_for_status(head) assert head.status_code == 200 self._session, self._url, self._chunk_size = session, url, chunk_size self._length = int(head.headers['Content-Length']) self._file = NamedTemporaryFile() self.truncate(self._length) self._left = [] # type: List[int] self._right = [] # type: List[int] if 'bytes' not in head.headers.get('Accept-Ranges', 'none'): raise HTTPRangeRequestUnsupported('range request is not supported') self._check_zip() @property def mode(self): # type: () -> str """Opening mode, which is always rb.""" return 'rb' @property def name(self): # type: () -> str """Path to the underlying file.""" return self._file.name def seekable(self): # type: () -> bool """Return whether random access is supported, which is True.""" return True def close(self): # type: () -> None """Close the file.""" self._file.close() @property def closed(self): # type: () -> bool """Whether the file is closed.""" return self._file.closed def read(self, size=-1): # type: (int) -> bytes """Read up to size bytes from the object and return them. As a convenience, if size is unspecified or -1, all bytes until EOF are returned. Fewer than size bytes may be returned if EOF is reached. """ download_size = max(size, self._chunk_size) start, length = self.tell(), self._length stop = length if size < 0 else min(start+download_size, length) start = max(0, stop-download_size) self._download(start, stop-1) return self._file.read(size) def readable(self): # type: () -> bool """Return whether the file is readable, which is True.""" return True def seek(self, offset, whence=0): # type: (int, int) -> int """Change stream position and return the new absolute position. Seek to offset relative position indicated by whence: * 0: Start of stream (the default). pos should be >= 0; * 1: Current position - pos may be negative; * 2: End of stream - pos usually negative. """ return self._file.seek(offset, whence) def tell(self): # type: () -> int """Return the current possition.""" return self._file.tell() def truncate(self, size=None): # type: (Optional[int]) -> int """Resize the stream to the given size in bytes. If size is unspecified resize to the current position. The current stream position isn't changed. Return the new file size. """ return self._file.truncate(size) def writable(self): # type: () -> bool """Return False.""" return False def __enter__(self): # type: () -> LazyZipOverHTTP self._file.__enter__() return self def __exit__(self, *exc): # type: (*Any) -> Optional[bool] return self._file.__exit__(*exc) @contextmanager def _stay(self): # type: ()-> Iterator[None] """Return a context manager keeping the position. At the end of the block, seek back to original position. """ pos = self.tell() try: yield finally: self.seek(pos) def _check_zip(self): # type: () -> None """Check and download until the file is a valid ZIP.""" end = self._length - 1 for start in reversed(range(0, end, self._chunk_size)): self._download(start, end) with self._stay(): try: # For read-only ZIP files, ZipFile only needs # methods read, seek, seekable and tell. ZipFile(self) # type: ignore except BadZipfile: pass else: break def _stream_response(self, start, end, base_headers=HEADERS): # type: (int, int, Dict[str, str]) -> Response """Return HTTP response to a range request from start to end."""